diff --git a/Documentation/Makefile b/Documentation/Makefile index 5c156fbb6cdf1e..3885bbe260eb23 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -106,10 +106,10 @@ YNL_RST_FILES_TMP := $(patsubst %.yaml,%.rst,$(wildcard $(YNL_YAML_DIR)/*.yaml)) YNL_RST_FILES := $(patsubst $(YNL_YAML_DIR)%,$(YNL_RST_DIR)%, $(YNL_RST_FILES_TMP)) $(YNL_INDEX): $(YNL_RST_FILES) - @$(YNL_TOOL) -o $@ -x + $(Q)$(YNL_TOOL) -o $@ -x -$(YNL_RST_DIR)/%.rst: $(YNL_YAML_DIR)/%.yaml - @$(YNL_TOOL) -i $< -o $@ +$(YNL_RST_DIR)/%.rst: $(YNL_YAML_DIR)/%.yaml $(YNL_TOOL) + $(Q)$(YNL_TOOL) -i $< -o $@ htmldocs: $(YNL_INDEX) @$(srctree)/scripts/sphinx-pre-install --version-check diff --git a/Documentation/admin-guide/sysctl/net.rst b/Documentation/admin-guide/sysctl/net.rst index c7525942f12ce9..3960916519557f 100644 --- a/Documentation/admin-guide/sysctl/net.rst +++ b/Documentation/admin-guide/sysctl/net.rst @@ -345,7 +345,10 @@ optmem_max ---------- Maximum ancillary buffer size allowed per socket. Ancillary data is a sequence -of struct cmsghdr structures with appended data. +of struct cmsghdr structures with appended data. TCP tx zerocopy also uses +optmem_max as a limit for its internal structures. + +Default : 128 KB fb_tunnels_only_for_init_net ---------------------------- diff --git a/Documentation/bpf/cpumasks.rst b/Documentation/bpf/cpumasks.rst index a22b6ad105fbb5..b5d47a04da5d8d 100644 --- a/Documentation/bpf/cpumasks.rst +++ b/Documentation/bpf/cpumasks.rst @@ -352,7 +352,7 @@ can be used to query the contents of cpumasks. .. kernel-doc:: kernel/bpf/cpumask.c :identifiers: bpf_cpumask_first bpf_cpumask_first_zero bpf_cpumask_first_and - bpf_cpumask_test_cpu + bpf_cpumask_test_cpu bpf_cpumask_weight .. kernel-doc:: kernel/bpf/cpumask.c :identifiers: bpf_cpumask_equal bpf_cpumask_intersects bpf_cpumask_subset diff --git a/Documentation/bpf/fs_kfuncs.rst b/Documentation/bpf/fs_kfuncs.rst new file mode 100644 index 00000000000000..8762c3233a3d18 --- /dev/null +++ b/Documentation/bpf/fs_kfuncs.rst @@ -0,0 +1,21 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _fs_kfuncs-header-label: + +===================== +BPF filesystem kfuncs +===================== + +BPF LSM programs need to access filesystem data from LSM hooks. The following +BPF kfuncs can be used to get these data. + + * ``bpf_get_file_xattr()`` + + * ``bpf_get_fsverity_digest()`` + +To avoid recursions, these kfuncs follow the following rules: + +1. These kfuncs are only permitted from BPF LSM function. +2. These kfuncs should not call into other LSM hooks, i.e. security_*(). For + example, ``bpf_get_file_xattr()`` does not use ``vfs_getxattr()``, because + the latter calls LSM hook ``security_inode_getxattr``. diff --git a/Documentation/bpf/index.rst b/Documentation/bpf/index.rst index aeaeb35e6d4a70..0bb5cb8157f133 100644 --- a/Documentation/bpf/index.rst +++ b/Documentation/bpf/index.rst @@ -21,6 +21,7 @@ that goes into great technical depth about the BPF Architecture. helpers kfuncs cpumasks + fs_kfuncs programs maps bpf_prog_run diff --git a/Documentation/devicetree/bindings/net/marvell,orion-mdio.yaml b/Documentation/devicetree/bindings/net/marvell,orion-mdio.yaml index e35da8b01dc257..73429855d5848f 100644 --- a/Documentation/devicetree/bindings/net/marvell,orion-mdio.yaml +++ b/Documentation/devicetree/bindings/net/marvell,orion-mdio.yaml @@ -39,28 +39,6 @@ required: allOf: - $ref: mdio.yaml# - - if: - required: - - interrupts - - then: - properties: - reg: - items: - - items: - - $ref: /schemas/types.yaml#/definitions/cell - - const: 0x84 - - else: - properties: - reg: - items: - - items: - - $ref: /schemas/types.yaml#/definitions/cell - - enum: - - 0x4 - - 0x10 - unevaluatedProperties: false examples: diff --git a/Documentation/driver-api/index.rst b/Documentation/driver-api/index.rst index f549a68951d70f..8bc4ebe7a36f85 100644 --- a/Documentation/driver-api/index.rst +++ b/Documentation/driver-api/index.rst @@ -115,6 +115,7 @@ available subsections can be seen below. hte/index wmi dpll + wbrf .. only:: subproject and html diff --git a/Documentation/driver-api/wbrf.rst b/Documentation/driver-api/wbrf.rst new file mode 100644 index 00000000000000..f48bfa02981348 --- /dev/null +++ b/Documentation/driver-api/wbrf.rst @@ -0,0 +1,78 @@ +.. SPDX-License-Identifier: GPL-2.0-or-later + +================================= +WBRF - Wifi Band RFI Mitigations +================================= + +Due to electrical and mechanical constraints in certain platform designs +there may be likely interference of relatively high-powered harmonics of +the GPU memory clocks with local radio module frequency bands used by +certain Wifi bands. + +To mitigate possible RFI interference producers can advertise the +frequencies in use and consumers can use this information to avoid using +these frequencies for sensitive features. + +When a platform is known to have this issue with any contained devices, +the platform designer will advertise the availability of this feature via +ACPI devices with a device specific method (_DSM). +* Producers with this _DSM will be able to advertise the frequencies in use. +* Consumers with this _DSM will be able to register for notifications of +frequencies in use. + +Some general terms +================== + +Producer: such component who can produce high-powered radio frequency +Consumer: such component who can adjust its in-use frequency in +response to the radio frequencies of other components to mitigate the +possible RFI. + +To make the mechanism function, those producers should notify active use +of their particular frequencies so that other consumers can make relative +internal adjustments as necessary to avoid this resonance. + +ACPI interface +============== + +Although initially used by for wifi + dGPU use cases, the ACPI interface +can be scaled to any type of device that a platform designer discovers +can cause interference. + +The GUID used for the _DSM is 7B7656CF-DC3D-4C1C-83E9-66E721DE3070. + +3 functions are available in this _DSM: + +* 0: discover # of functions available +* 1: record RF bands in use +* 2: retrieve RF bands in use + +Driver programming interface +============================ + +.. kernel-doc:: drivers/platform/x86/amd/wbrf.c + +Sample Usage +============= + +The expected flow for the producers: +1. During probe, call `acpi_amd_wbrf_supported_producer` to check if WBRF +can be enabled for the device. +2. On using some frequency band, call `acpi_amd_wbrf_add_remove` with 'add' +param to get other consumers properly notified. +3. Or on stopping using some frequency band, call +`acpi_amd_wbrf_add_remove` with 'remove' param to get other consumers notified. + +The expected flow for the consumers: +1. During probe, call `acpi_amd_wbrf_supported_consumer` to check if WBRF +can be enabled for the device. +2. Call `amd_wbrf_register_notifier` to register for notification +of frequency band change(add or remove) from other producers. +3. Call the `amd_wbrf_retrieve_freq_band` initally to retrieve +current active frequency bands considering some producers may broadcast +such information before the consumer is up. +4. On receiving a notification for frequency band change, run +`amd_wbrf_retrieve_freq_band` again to retrieve the latest +active frequency bands. +5. During driver cleanup, call `amd_wbrf_unregister_notifier` to +unregister the notifier. diff --git a/Documentation/netlink/netlink-raw.yaml b/Documentation/netlink/netlink-raw.yaml index ad5395040765cf..04b92f1a5cd6ef 100644 --- a/Documentation/netlink/netlink-raw.yaml +++ b/Documentation/netlink/netlink-raw.yaml @@ -126,8 +126,10 @@ properties: name: type: string type: - description: The netlink attribute type - enum: [ u8, u16, u32, u64, s8, s16, s32, s64, string, binary ] + description: | + The netlink attribute type. Members of type 'binary' or 'pad' + must also have the 'len' property set. + enum: [ u8, u16, u32, u64, s8, s16, s32, s64, string, binary, pad ] len: $ref: '#/$defs/len-or-define' byte-order: @@ -150,6 +152,14 @@ properties: the right formatting mechanism when displaying values of this type. enum: [ hex, mac, fddi, ipv4, ipv6, uuid ] + if: + properties: + type: + oneOf: + - const: binary + - const: pad + then: + required: [ len ] # End genetlink-legacy attribute-sets: @@ -202,7 +212,8 @@ properties: description: The netlink attribute type enum: [ unused, pad, flag, binary, bitfield32, u8, u16, u32, u64, s8, s16, s32, s64, - string, nest, array-nest, nest-type-value ] + string, nest, array-nest, nest-type-value, + sub-message ] doc: description: Documentation of the attribute. type: string @@ -261,6 +272,17 @@ properties: description: Name of the struct type used for the attribute. type: string # End genetlink-legacy + # Start netlink-raw + sub-message: + description: | + Name of the sub-message definition to use for the attribute. + type: string + selector: + description: | + Name of the attribute to use for dynamic selection of sub-message + format specifier. + type: string + # End netlink-raw # Make sure name-prefix does not appear in subsets (subsets inherit naming) dependencies: @@ -283,6 +305,43 @@ properties: items: required: [ type ] + # Start netlink-raw + sub-messages: + description: Definition of sub message attributes + type: array + items: + type: object + additionalProperties: False + required: [ name, formats ] + properties: + name: + description: Name of the sub-message definition + type: string + formats: + description: Dynamically selected format specifiers + type: array + items: + type: object + additionalProperties: False + required: [ value ] + properties: + value: + description: | + Value to match for dynamic selection of sub-message format + specifier. + type: string + fixed-header: + description: | + Name of the struct definition to use as the fixed header + for the sub message. + type: string + attribute-set: + description: | + Name of the attribute space from which to resolve attributes + in the sub message. + type: string + # End netlink-raw + operations: description: Operations supported by the protocol. type: object diff --git a/Documentation/netlink/specs/mptcp.yaml b/Documentation/netlink/specs/mptcp_pm.yaml similarity index 100% rename from Documentation/netlink/specs/mptcp.yaml rename to Documentation/netlink/specs/mptcp_pm.yaml diff --git a/Documentation/netlink/specs/netdev.yaml b/Documentation/netlink/specs/netdev.yaml index f2c76d103bd86a..3addac97068048 100644 --- a/Documentation/netlink/specs/netdev.yaml +++ b/Documentation/netlink/specs/netdev.yaml @@ -54,6 +54,10 @@ definitions: name: hash doc: Device is capable of exposing receive packet hash via bpf_xdp_metadata_rx_hash(). + - + name: vlan-tag + doc: + Device is capable of exposing receive packet VLAN tag via bpf_xdp_metadata_rx_vlan_tag(). - type: flags name: xsk-flags diff --git a/Documentation/netlink/specs/ovs_datapath.yaml b/Documentation/netlink/specs/ovs_datapath.yaml index f709c26c3e92d4..edc8c95ca6f5ca 100644 --- a/Documentation/netlink/specs/ovs_datapath.yaml +++ b/Documentation/netlink/specs/ovs_datapath.yaml @@ -20,6 +20,7 @@ definitions: name: user-features type: flags name-prefix: ovs-dp-f- + enum-name: entries: - name: unaligned @@ -142,7 +143,6 @@ operations: do: request: attributes: - - dp-ifindex - name - upcall-pid - user-features @@ -154,7 +154,6 @@ operations: do: request: attributes: - - dp-ifindex - name mcast-groups: diff --git a/Documentation/netlink/specs/ovs_flow.yaml b/Documentation/netlink/specs/ovs_flow.yaml index 109ca1f57b6c6f..4fdfc6b5cae952 100644 --- a/Documentation/netlink/specs/ovs_flow.yaml +++ b/Documentation/netlink/specs/ovs_flow.yaml @@ -124,6 +124,7 @@ definitions: - name: ovs-frag-type name-prefix: ovs-frag-type- + enum-name: ovs-frag-type type: enum entries: - @@ -269,6 +270,7 @@ definitions: - name: ovs-ufid-flags name-prefix: ovs-ufid-f- + enum-name: type: flags entries: - omit-key @@ -288,6 +290,7 @@ definitions: doc: Basis used for computing hash. - name: ovs-hash-alg + enum-name: ovs-hash-alg type: enum doc: | Data path hash algorithm for computing Datapath hash. The algorithm type only specifies @@ -339,6 +342,7 @@ definitions: MPLS tunnel attributes. - name: ct-state-flags + enum-name: type: flags name-prefix: ovs-cs-f- entries: @@ -947,13 +951,11 @@ operations: do: &flow-get-op request: attributes: - - dp-ifindex - key - ufid - ufid-flags reply: attributes: - - dp-ifindex - key - ufid - mask @@ -968,7 +970,6 @@ operations: do: request: attributes: - - dp-ifindex - key - ufid - mask diff --git a/Documentation/netlink/specs/ovs_vport.yaml b/Documentation/netlink/specs/ovs_vport.yaml index f65ce62cd60de1..86ba9ac2a52103 100644 --- a/Documentation/netlink/specs/ovs_vport.yaml +++ b/Documentation/netlink/specs/ovs_vport.yaml @@ -135,7 +135,6 @@ operations: - name - type - upcall-pid - - dp-ifindex - ifindex - options - @@ -146,7 +145,6 @@ operations: do: request: attributes: - - dp-ifindex - port-no - type - name @@ -158,11 +156,9 @@ operations: do: &vport-get-op request: attributes: - - dp-ifindex - name reply: &dev-all attributes: - - dp-ifindex - port-no - type - name diff --git a/Documentation/netlink/specs/rt_link.yaml b/Documentation/netlink/specs/rt_link.yaml index d86a68f8475ca8..1ad01d52a8638d 100644 --- a/Documentation/netlink/specs/rt_link.yaml +++ b/Documentation/netlink/specs/rt_link.yaml @@ -66,8 +66,9 @@ definitions: name: ifi-family type: u8 - - name: padding - type: u8 + name: pad + type: pad + len: 1 - name: ifi-type type: u16 @@ -82,6 +83,18 @@ definitions: - name: ifi-change type: u32 + - + name: ifla-bridge-id + type: struct + members: + - + name: prio + type: u16 + - + name: addr + type: binary + len: 6 + display-hint: mac - name: ifla-cacheinfo type: struct @@ -707,11 +720,9 @@ definitions: name: family type: u8 - - name: pad1 - type: u8 - - - name: pad2 - type: u16 + name: pad + type: pad + len: 3 - name: ifindex type: u32 @@ -966,8 +977,9 @@ attribute-sets: type: string - name: data - type: binary - # kind specific nest, e.g. linkinfo-bridge-attrs + type: sub-message + sub-message: linkinfo-data-msg + selector: kind - name: xstats type: binary @@ -976,10 +988,12 @@ attribute-sets: type: string - name: slave-data - type: binary - # kind specific nest + type: sub-message + sub-message: linkinfo-member-data-msg + selector: slave-kind - name: linkinfo-bridge-attrs + name-prefix: ifla-br- attributes: - name: forward-delay @@ -1011,9 +1025,11 @@ attribute-sets: - name: root-id type: binary + struct: ifla-bridge-id - name: bridge-id type: binary + struct: ifla-bridge-id - name: root-port type: u16 @@ -1041,6 +1057,7 @@ attribute-sets: - name: group-addr type: binary + display-hint: mac - name: fdb-flush type: binary @@ -1123,6 +1140,376 @@ attribute-sets: - name: mcast-querier-state type: binary + - + name: linkinfo-brport-attrs + name-prefix: ifla-brport- + attributes: + - + name: state + type: u8 + - + name: priority + type: u16 + - + name: cost + type: u32 + - + name: mode + type: flag + - + name: guard + type: flag + - + name: protect + type: flag + - + name: fast-leave + type: flag + - + name: learning + type: flag + - + name: unicast-flood + type: flag + - + name: proxyarp + type: flag + - + name: learning-sync + type: flag + - + name: proxyarp-wifi + type: flag + - + name: root-id + type: binary + struct: ifla-bridge-id + - + name: bridge-id + type: binary + struct: ifla-bridge-id + - + name: designated-port + type: u16 + - + name: designated-cost + type: u16 + - + name: id + type: u16 + - + name: "no" + type: u16 + - + name: topology-change-ack + type: u8 + - + name: config-pending + type: u8 + - + name: message-age-timer + type: u64 + - + name: forward-delay-timer + type: u64 + - + name: hold-timer + type: u64 + - + name: flush + type: flag + - + name: multicast-router + type: u8 + - + name: pad + type: pad + - + name: mcast-flood + type: flag + - + name: mcast-to-ucast + type: flag + - + name: vlan-tunnel + type: flag + - + name: bcast-flood + type: flag + - + name: group-fwd-mask + type: u16 + - + name: neigh-suppress + type: flag + - + name: isolated + type: flag + - + name: backup-port + type: u32 + - + name: mrp-ring-open + type: flag + - + name: mrp-in-open + type: flag + - + name: mcast-eht-hosts-limit + type: u32 + - + name: mcast-eht-hosts-cnt + type: u32 + - + name: locked + type: flag + - + name: mab + type: flag + - + name: mcast-n-groups + type: u32 + - + name: mcast-max-groups + type: u32 + - + name: neigh-vlan-suppress + type: flag + - + name: backup-nhid + type: u32 + - + name: linkinfo-gre-attrs + name-prefix: ifla-gre- + attributes: + - + name: link + type: u32 + - + name: iflags + type: u16 + - + name: oflags + type: u16 + - + name: ikey + type: u32 + - + name: okey + type: u32 + - + name: local + type: binary + display-hint: ipv4 + - + name: remote + type: binary + display-hint: ipv4 + - + name: ttl + type: u8 + - + name: tos + type: u8 + - + name: pmtudisc + type: u8 + - + name: encap-limit + type: u32 + - + name: flowinfo + type: u32 + - + name: flags + type: u32 + - + name: encap-type + type: u16 + - + name: encap-flags + type: u16 + - + name: encap-sport + type: u16 + - + name: encap-dport + type: u16 + - + name: collect-metadata + type: flag + - + name: ignore-df + type: u8 + - + name: fwmark + type: u32 + - + name: erspan-index + type: u32 + - + name: erspan-ver + type: u8 + - + name: erspan-dir + type: u8 + - + name: erspan-hwid + type: u16 + - + name: linkinfo-geneve-attrs + name-prefix: ifla-geneve- + attributes: + - + name: id + type: u32 + - + name: remote + type: binary + display-hint: ipv4 + - + name: ttl + type: u8 + - + name: tos + type: u8 + - + name: port + type: u16 + - + name: collect-metadata + type: flag + - + name: remote6 + type: binary + display-hint: ipv6 + - + name: udp-csum + type: u8 + - + name: udp-zero-csum6-tx + type: u8 + - + name: udp-zero-csum6-rx + type: u8 + - + name: label + type: u32 + - + name: ttl-inherit + type: u8 + - + name: df + type: u8 + - + name: inner-proto-inherit + type: flag + - + name: linkinfo-iptun-attrs + name-prefix: ifla-iptun- + attributes: + - + name: link + type: u32 + - + name: local + type: binary + display-hint: ipv4 + - + name: remote + type: binary + display-hint: ipv4 + - + name: ttl + type: u8 + - + name: tos + type: u8 + - + name: encap-limit + type: u8 + - + name: flowinfo + type: u32 + - + name: flags + type: u16 + - + name: proto + type: u8 + - + name: pmtudisc + type: u8 + - + name: 6rd-prefix + type: binary + display-hint: ipv6 + - + name: 6rd-relay-prefix + type: binary + display-hint: ipv4 + - + name: 6rd-prefixlen + type: u16 + - + name: 6rd-relay-prefixlen + type: u16 + - + name: encap-type + type: u16 + - + name: encap-flags + type: u16 + - + name: encap-sport + type: u16 + - + name: encap-dport + type: u16 + - + name: collect-metadata + type: flag + - + name: fwmark + type: u32 + - + name: linkinfo-tun-attrs + name-prefix: ifla-tun- + attributes: + - + name: owner + type: u32 + - + name: group + type: u32 + - + name: type + type: u8 + - + name: pi + type: u8 + - + name: vnet-hdr + type: u8 + - + name: persist + type: u8 + - + name: multi-queue + type: u8 + - + name: num-queues + type: u32 + - + name: num-disabled-queues + type: u32 + - + name: linkinfo-vrf-attrs + name-prefix: ifla-vrf- + attributes: + - + name: table + type: u32 - name: xdp-attrs attributes: @@ -1241,6 +1628,46 @@ attribute-sets: name: used type: u8 +sub-messages: + - + name: linkinfo-data-msg + formats: + - + value: bridge + attribute-set: linkinfo-bridge-attrs + - + value: erspan + attribute-set: linkinfo-gre-attrs + - + value: gre + attribute-set: linkinfo-gre-attrs + - + value: gretap + attribute-set: linkinfo-gre-attrs + - + value: geneve + attribute-set: linkinfo-geneve-attrs + - + value: ipip + attribute-set: linkinfo-iptun-attrs + - + value: sit + attribute-set: linkinfo-iptun-attrs + - + value: tun + attribute-set: linkinfo-tun-attrs + - + value: vrf + attribute-set: linkinfo-vrf-attrs + - + name: linkinfo-member-data-msg + formats: + - + value: bridge + attribute-set: linkinfo-brport-attrs + - + value: bond + operations: enum-model: directional list: diff --git a/Documentation/netlink/specs/tc.yaml b/Documentation/netlink/specs/tc.yaml new file mode 100644 index 00000000000000..4346fa402fc91d --- /dev/null +++ b/Documentation/netlink/specs/tc.yaml @@ -0,0 +1,2031 @@ +# SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) + +name: tc +protocol: netlink-raw +protonum: 0 + +doc: + Netlink raw family for tc qdisc, chain, class and filter configuration + over rtnetlink. + +definitions: + - + name: tcmsg + type: struct + members: + - + name: family + type: u8 + - + name: pad + type: pad + len: 3 + - + name: ifindex + type: s32 + - + name: handle + type: u32 + - + name: parent + type: u32 + - + name: info + type: u32 + - + name: tc-cls-flags + type: flags + entries: + - skip-hw + - skip-sw + - in-hw + - not-in-nw + - verbose + - + name: tc-stats + type: struct + members: + - + name: bytes + type: u64 + - + name: packets + type: u32 + - + name: drops + type: u32 + - + name: overlimits + type: u32 + - + name: bps + type: u32 + - + name: pps + type: u32 + - + name: qlen + type: u32 + - + name: backlog + type: u32 + - + name: tc-cbs-qopt + type: struct + members: + - + name: offload + type: u8 + - + name: pad + type: pad + len: 3 + - + name: hicredit + type: s32 + - + name: locredit + type: s32 + - + name: idleslope + type: s32 + - + name: sendslope + type: s32 + - + name: tc-etf-qopt + type: struct + members: + - + name: delta + type: s32 + - + name: clockid + type: s32 + - + name: flags + type: s32 + - + name: tc-fifo-qopt + type: struct + members: + - + name: limit + type: u32 + - + name: tc-htb-opt + type: struct + members: + - + name: rate + type: binary + len: 12 + - + name: ceil + type: binary + len: 12 + - + name: buffer + type: u32 + - + name: cbuffer + type: u32 + - + name: quantum + type: u32 + - + name: level + type: u32 + - + name: prio + type: u32 + - + name: tc-htb-glob + type: struct + members: + - + name: version + type: u32 + - + name: rate2quantum + type: u32 + - + name: defcls + type: u32 + - + name: debug + type: u32 + - + name: direct-pkts + type: u32 + - + name: tc-gred-qopt + type: struct + members: + - + name: limit + type: u32 + - + name: qth-min + type: u32 + - + name: qth-max + type: u32 + - + name: DP + type: u32 + - + name: backlog + type: u32 + - + name: qave + type: u32 + - + name: forced + type: u32 + - + name: early + type: u32 + - + name: other + type: u32 + - + name: pdrop + type: u32 + - + name: Wlog + type: u8 + - + name: Plog + type: u8 + - + name: Scell_log + type: u8 + - + name: prio + type: u8 + - + name: packets + type: u32 + - + name: bytesin + type: u32 + - + name: tc-gred-sopt + type: struct + members: + - + name: DPs + type: u32 + - + name: def_DP + type: u32 + - + name: grio + type: u8 + - + name: flags + type: u8 + - + name: pad + type: pad + len: 2 + - + name: tc-hfsc-qopt + type: struct + members: + - + name: defcls + type: u16 + - + name: tc-mqprio-qopt + type: struct + members: + - + name: num-tc + type: u8 + - + name: prio-tc-map + type: binary + len: 16 + - + name: hw + type: u8 + - + name: count + type: binary + len: 32 + - + name: offset + type: binary + len: 32 + - + name: tc-multiq-qopt + type: struct + members: + - + name: bands + type: u16 + - + name: max-bands + type: u16 + - + name: tc-netem-qopt + type: struct + members: + - + name: latency + type: u32 + - + name: limit + type: u32 + - + name: loss + type: u32 + - + name: gap + type: u32 + - + name: duplicate + type: u32 + - + name: jitter + type: u32 + - + name: tc-plug-qopt + type: struct + members: + - + name: action + type: s32 + - + name: limit + type: u32 + - + name: tc-prio-qopt + type: struct + members: + - + name: bands + type: u16 + - + name: priomap + type: binary + len: 16 + - + name: tc-red-qopt + type: struct + members: + - + name: limit + type: u32 + - + name: qth-min + type: u32 + - + name: qth-max + type: u32 + - + name: Wlog + type: u8 + - + name: Plog + type: u8 + - + name: Scell-log + type: u8 + - + name: flags + type: u8 + - + name: tc-sfb-qopt + type: struct + members: + - + name: rehash-interval + type: u32 + - + name: warmup-time + type: u32 + - + name: max + type: u32 + - + name: bin-size + type: u32 + - + name: increment + type: u32 + - + name: decrement + type: u32 + - + name: limit + type: u32 + - + name: penalty-rate + type: u32 + - + name: penalty-burst + type: u32 + - + name: tc-sfq-qopt-v1 # TODO nested structs + type: struct + members: + - + name: quantum + type: u32 + - + name: perturb-period + type: s32 + - + name: limit + type: u32 + - + name: divisor + type: u32 + - + name: flows + type: u32 + - + name: depth + type: u32 + - + name: headdrop + type: u32 + - + name: limit + type: u32 + - + name: qth-min + type: u32 + - + name: qth-mac + type: u32 + - + name: Wlog + type: u8 + - + name: Plog + type: u8 + - + name: Scell-log + type: u8 + - + name: flags + type: u8 + - + name: max-P + type: u32 + - + name: prob-drop + type: u32 + - + name: forced-drop + type: u32 + - + name: prob-mark + type: u32 + - + name: forced-mark + type: u32 + - + name: prob-mark-head + type: u32 + - + name: forced-mark-head + type: u32 + - + name: tc-tbf-qopt + type: struct + members: + - + name: rate + type: binary # TODO nested struct tc_ratespec + len: 12 + - + name: peakrate + type: binary # TODO nested struct tc_ratespec + len: 12 + - + name: limit + type: u32 + - + name: buffer + type: u32 + - + name: mtu + type: u32 + - + name: tc-sizespec + type: struct + members: + - + name: cell-log + type: u8 + - + name: size-log + type: u8 + - + name: cell-align + type: s16 + - + name: overhead + type: s32 + - + name: linklayer + type: u32 + - + name: mpu + type: u32 + - + name: mtu + type: u32 + - + name: tsize + type: u32 + - + name: gnet-estimator + type: struct + members: + - + name: interval + type: s8 + - + name: ewma-log + type: u8 +attribute-sets: + - + name: tc-attrs + attributes: + - + name: kind + type: string + - + name: options + type: sub-message + sub-message: tc-options-msg + selector: kind + - + name: stats + type: binary + struct: tc-stats + - + name: xstats + type: binary + - + name: rate + type: binary + struct: gnet-estimator + - + name: fcnt + type: u32 + - + name: stats2 + type: nest + nested-attributes: tca-stats-attrs + - + name: stab + type: nest + nested-attributes: tca-stab-attrs + - + name: pad + type: pad + - + name: dump-invisible + type: flag + - + name: chain + type: u32 + - + name: hw-offload + type: u8 + - + name: ingress-block + type: u32 + - + name: egress-block + type: u32 + - + name: dump-flags + type: bitfield32 + - + name: ext-warn-msg + type: string + - + name: tc-cake-attrs + attributes: + - + name: pad + type: pad + - + name: base-rate64 + type: u64 + - + name: diffserv-mode + type: u32 + - + name: atm + type: u32 + - + name: flow-mode + type: u32 + - + name: overhead + type: u32 + - + name: rtt + type: u32 + - + name: target + type: u32 + - + name: autorate + type: u32 + - + name: memory + type: u32 + - + name: nat + type: u32 + - + name: raw + type: u32 + - + name: wash + type: u32 + - + name: mpu + type: u32 + - + name: ingress + type: u32 + - + name: ack-filter + type: u32 + - + name: split-gso + type: u32 + - + name: fwmark + type: u32 + - + name: tc-cake-stats-attrs + attributes: + - + name: pad + type: pad + - + name: capacity-estimate64 + type: u64 + - + name: memory-limit + type: u32 + - + name: memory-used + type: u32 + - + name: avg-netoff + type: u32 + - + name: min-netlen + type: u32 + - + name: max-netlen + type: u32 + - + name: min-adjlen + type: u32 + - + name: max-adjlen + type: u32 + - + name: tin-stats + type: binary + - + name: deficit + type: s32 + - + name: cobalt-count + type: u32 + - + name: dropping + type: u32 + - + name: drop-next-us + type: s32 + - + name: p-drop + type: u32 + - + name: blue-timer-us + type: s32 + - + name: tc-cbs-attrs + attributes: + - + name: parms + type: binary + struct: tc-cbs-qopt + - + name: tc-choke-attrs + attributes: + - + name: parms + type: binary + struct: tc-red-qopt + - + name: stab + type: binary + - + name: max-p + type: u32 + - + name: tc-codel-attrs + attributes: + - + name: target + type: u32 + - + name: limit + type: u32 + - + name: interval + type: u32 + - + name: ecn + type: u32 + - + name: ce-threshold + type: u32 + - + name: tc-drr-attrs + attributes: + - + name: quantum + type: u32 + - + name: tc-flower-attrs + attributes: + - + name: classid + type: u32 + - + name: indev + type: string + - + name: act + type: array-nest + nested-attributes: tc-act-attrs + - + name: key-eth-dst + type: binary + display-hint: mac + - + name: key-eth-dst-mask + type: binary + display-hint: mac + - + name: key-eth-src + type: binary + display-hint: mac + - + name: key-eth-src-mask + type: binary + display-hint: mac + - + name: key-eth-type + type: u16 + byte-order: big-endian + - + name: key-ip-proto + type: u8 + - + name: key-ipv4-src + type: u32 + byte-order: big-endian + display-hint: ipv4 + - + name: key-ipv4-src-mask + type: u32 + byte-order: big-endian + display-hint: ipv4 + - + name: key-ipv4-dst + type: u32 + byte-order: big-endian + display-hint: ipv4 + - + name: key-ipv4-dst-mask + type: u32 + byte-order: big-endian + display-hint: ipv4 + - + name: key-ipv6-src + type: binary + display-hint: ipv6 + - + name: key-ipv6-src-mask + type: binary + display-hint: ipv6 + - + name: key-ipv6-dst + type: binary + display-hint: ipv6 + - + name: key-ipv6-dst-mask + type: binary + display-hint: ipv6 + - + name: key-tcp-src + type: u16 + byte-order: big-endian + - + name: key-tcp-dst + type: u16 + byte-order: big-endian + - + name: key-udp-src + type: u16 + byte-order: big-endian + - + name: key-udp-dst + type: u16 + byte-order: big-endian + - + name: flags + type: u32 + enum: tc-cls-flags + enum-as-flags: true + - + name: key-vlan-id + type: u16 + byte-order: big-endian + - + name: key-vlan-prio + type: u8 + - + name: key-vlan-eth-type + type: u16 + byte-order: big-endian + - + name: key-enc-key-id + type: u32 + byte-order: big-endian + - + name: key-enc-ipv4-src + type: u32 + byte-order: big-endian + display-hint: ipv4 + - + name: key-enc-ipv4-src-mask + type: u32 + byte-order: big-endian + display-hint: ipv4 + - + name: key-enc-ipv4-dst + type: u32 + byte-order: big-endian + display-hint: ipv4 + - + name: key-enc-ipv4-dst-mask + type: u32 + byte-order: big-endian + display-hint: ipv4 + - + name: key-enc-ipv6-src + type: binary + display-hint: ipv6 + - + name: key-enc-ipv6-src-mask + type: binary + display-hint: ipv6 + - + name: key-enc-ipv6-dst + type: binary + display-hint: ipv6 + - + name: key-enc-ipv6-dst-mask + type: binary + display-hint: ipv6 + - + name: key-tcp-src-mask + type: u16 + byte-order: big-endian + - + name: key-tcp-dst-mask + type: u16 + byte-order: big-endian + - + name: key-udp-src-mask + type: u16 + byte-order: big-endian + - + name: key-udp-dst-mask + type: u16 + byte-order: big-endian + - + name: key-sctp-src-mask + type: u16 + byte-order: big-endian + - + name: key-sctp-dst-mask + type: u16 + byte-order: big-endian + - + name: key-sctp-src + type: u16 + byte-order: big-endian + - + name: key-sctp-dst + type: u16 + byte-order: big-endian + - + name: key-enc-udp-src-port + type: u16 + byte-order: big-endian + - + name: key-enc-udp-src-port-mask + type: u16 + byte-order: big-endian + - + name: key-enc-udp-dst-port + type: u16 + byte-order: big-endian + - + name: key-enc-udp-dst-port-mask + type: u16 + byte-order: big-endian + - + name: key-flags + type: u32 + byte-order: big-endian + - + name: key-flags-mask + type: u32 + byte-order: big-endian + - + name: key-icmpv4-code + type: u8 + - + name: key-icmpv4-code-mask + type: u8 + - + name: key-icmpv4-type + type: u8 + - + name: key-icmpv4-type-mask + type: u8 + - + name: key-icmpv6-code + type: u8 + - + name: key-icmpv6-code-mask + type: u8 + - + name: key-icmpv6-type + type: u8 + - + name: key-icmpv6-type-mask + type: u8 + - + name: key-arp-sip + type: u32 + byte-order: big-endian + - + name: key-arp-sip-mask + type: u32 + byte-order: big-endian + - + name: key-arp-tip + type: u32 + byte-order: big-endian + - + name: key-arp-tip-mask + type: u32 + byte-order: big-endian + - + name: key-arp-op + type: u8 + - + name: key-arp-op-mask + type: u8 + - + name: key-arp-sha + type: binary + - + name: key-arp-sha-mask + type: binary + - + name: key-arp-tha + type: binary + - + name: key-arp-tha-mask + type: binary + - + name: key-mpls-ttl + type: u8 + - + name: key-mpls-bos + type: u8 + - + name: key-mpls-tc + type: u8 + - + name: key-mpls-label + type: u32 + byte-order: big-endian + - + name: key-tcp-flags + type: u16 + byte-order: big-endian + - + name: key-tcp-flags-mask + type: u16 + byte-order: big-endian + - + name: key-ip-tos + type: u8 + - + name: key-ip-tos-mask + type: u8 + - + name: key-ip-ttl + type: u8 + - + name: key-ip-ttl-mask + type: u8 + - + name: key-cvlan-id + type: u16 + byte-order: big-endian + - + name: key-cvlan-prio + type: u8 + - + name: key-cvlan-eth-type + type: u16 + byte-order: big-endian + - + name: key-enc-ip-tos + type: u8 + - + name: key-enc-ip-tos-mask + type: u8 + - + name: key-enc-ip-ttl + type: u8 + - + name: key-enc-ip-ttl-mask + type: u8 + - + name: key-enc-opts + type: binary + - + name: key-enc-opts-mask + type: binary + - + name: in-hw-count + type: u32 + - + name: key-port-src-min + type: u16 + byte-order: big-endian + - + name: key-port-src-max + type: u16 + byte-order: big-endian + - + name: key-port-dst-min + type: u16 + byte-order: big-endian + - + name: key-port-dst-max + type: u16 + byte-order: big-endian + - + name: key-ct-state + type: u16 + - + name: key-ct-state-mask + type: u16 + - + name: key-ct-zone + type: u16 + - + name: key-ct-zone-mask + type: u16 + - + name: key-ct-mark + type: u32 + - + name: key-ct-mark-mask + type: u32 + - + name: key-ct-labels + type: binary + - + name: key-ct-labels-mask + type: binary + - + name: key-mpls-opts + type: binary + - + name: key-hash + type: u32 + - + name: key-hash-mask + type: u32 + - + name: key-num-of-vlans + type: u8 + - + name: key-pppoe-sid + type: u16 + byte-order: big-endian + - + name: key-ppp-proto + type: u16 + byte-order: big-endian + - + name: key-l2-tpv3-sid + type: u32 + byte-order: big-endian + - + name: tc-gred-attrs + attributes: + - + name: parms + type: binary # array of struct: tc-gred-qopt + - + name: stab + type: binary + sub-type: u8 + - + name: dps + type: binary + struct: tc-gred-sopt + - + name: max-p + type: binary + sub-type: u32 + - + name: limit + type: u32 + - + name: vq-list + type: nest + nested-attributes: tca-gred-vq-list-attrs + - + name: tca-gred-vq-list-attrs + attributes: + - + name: entry + type: nest + nested-attributes: tca-gred-vq-entry-attrs + multi-attr: true + - + name: tca-gred-vq-entry-attrs + attributes: + - + name: pad + type: pad + - + name: dp + type: u32 + - + name: stat-bytes + type: u32 + - + name: stat-packets + type: u32 + - + name: stat-backlog + type: u32 + - + name: stat-prob-drop + type: u32 + - + name: stat-prob-mark + type: u32 + - + name: stat-forced-drop + type: u32 + - + name: stat-forced-mark + type: u32 + - + name: stat-pdrop + type: u32 + - + name: stat-other + type: u32 + - + name: flags + type: u32 + - + name: tc-hfsc-attrs + attributes: + - + name: rsc + type: binary + - + name: fsc + type: binary + - + name: usc + type: binary + - + name: tc-hhf-attrs + attributes: + - + name: backlog-limit + type: u32 + - + name: quantum + type: u32 + - + name: hh-flows-limit + type: u32 + - + name: reset-timeout + type: u32 + - + name: admit-bytes + type: u32 + - + name: evict-timeout + type: u32 + - + name: non-hh-weight + type: u32 + - + name: tc-htb-attrs + attributes: + - + name: parms + type: binary + struct: tc-htb-opt + - + name: init + type: binary + struct: tc-htb-glob + - + name: ctab + type: binary + - + name: rtab + type: binary + - + name: direct-qlen + type: u32 + - + name: rate64 + type: u64 + - + name: ceil64 + type: u64 + - + name: pad + type: pad + - + name: offload + type: flag + - + name: tc-act-attrs + attributes: + - + name: kind + type: string + - + name: options + type: sub-message + sub-message: tc-act-options-msg + selector: kind + - + name: index + type: u32 + - + name: stats + type: binary + - + name: pad + type: pad + - + name: cookie + type: binary + - + name: flags + type: bitfield32 + - + name: hw-stats + type: bitfield32 + - + name: used-hw-stats + type: bitfield32 + - + name: in-hw-count + type: u32 + - + name: tc-etf-attrs + attributes: + - + name: parms + type: binary + struct: tc-etf-qopt + - + name: tc-ets-attrs + attributes: + - + name: nbands + type: u8 + - + name: nstrict + type: u8 + - + name: quanta + type: nest + nested-attributes: tc-ets-attrs + - + name: quanta-band + type: u32 + multi-attr: true + - + name: priomap + type: nest + nested-attributes: tc-ets-attrs + - + name: priomap-band + type: u8 + multi-attr: true + - + name: tc-fq-attrs + attributes: + - + name: plimit + type: u32 + - + name: flow-plimit + type: u32 + - + name: quantum + type: u32 + - + name: initial-quantum + type: u32 + - + name: rate-enable + type: u32 + - + name: flow-default-rate + type: u32 + - + name: flow-max-rate + type: u32 + - + name: buckets-log + type: u32 + - + name: flow-refill-delay + type: u32 + - + name: orphan-mask + type: u32 + - + name: low-rate-threshold + type: u32 + - + name: ce-threshold + type: u32 + - + name: timer-slack + type: u32 + - + name: horizon + type: u32 + - + name: horizon-drop + type: u8 + - + name: tc-fq-codel-attrs + attributes: + - + name: target + type: u32 + - + name: limit + type: u32 + - + name: interval + type: u32 + - + name: ecn + type: u32 + - + name: flows + type: u32 + - + name: quantum + type: u32 + - + name: ce-threshold + type: u32 + - + name: drop-batch-size + type: u32 + - + name: memory-limit + type: u32 + - + name: ce-threshold-selector + type: u8 + - + name: ce-threshold-mask + type: u8 + - + name: tc-fq-pie-attrs + attributes: + - + name: limit + type: u32 + - + name: flows + type: u32 + - + name: target + type: u32 + - + name: tupdate + type: u32 + - + name: alpha + type: u32 + - + name: beta + type: u32 + - + name: quantum + type: u32 + - + name: memory-limit + type: u32 + - + name: ecn-prob + type: u32 + - + name: ecn + type: u32 + - + name: bytemode + type: u32 + - + name: dq-rate-estimator + type: u32 + - + name: tc-netem-attrs + attributes: + - + name: corr + type: binary + - + name: delay-dist + type: binary + sub-type: s16 + - + name: reorder + type: binary + - + name: corrupt + type: binary + - + name: loss + type: binary + - + name: rate + type: binary + - + name: ecn + type: u32 + - + name: rate64 + type: u64 + - + name: pad + type: u32 + - + name: latency64 + type: s64 + - + name: jitter64 + type: s64 + - + name: slot + type: binary + - + name: slot-dist + type: binary + sub-type: s16 + - + name: tc-pie-attrs + attributes: + - + name: target + type: u32 + - + name: limit + type: u32 + - + name: tupdate + type: u32 + - + name: alpha + type: u32 + - + name: beta + type: u32 + - + name: ecn + type: u32 + - + name: bytemode + type: u32 + - + name: dq-rate-estimator + type: u32 + - + name: tc-qfq-attrs + attributes: + - + name: weight + type: u32 + - + name: lmax + type: u32 + - + name: tc-red-attrs + attributes: + - + name: parms + type: binary + struct: tc-red-qopt + - + name: stab + type: binary + - + name: max-p + type: u32 + - + name: flags + type: binary + - + name: early-drop-block + type: u32 + - + name: mark-block + type: u32 + - + name: tc-taprio-attrs + attributes: + - + name: priomap + type: binary + struct: tc-mqprio-qopt + - + name: sched-entry-list + type: nest + nested-attributes: tc-taprio-sched-entry-list + - + name: sched-base-time + type: s64 + - + name: sched-single-entry + type: nest + nested-attributes: tc-taprio-sched-entry + - + name: sched-clockid + type: s32 + - + name: pad + type: pad + - + name: admin-sched + type: binary + - + name: sched-cycle-time + type: s64 + - + name: sched-cycle-time-extension + type: s64 + - + name: flags + type: u32 + - + name: txtime-delay + type: u32 + - + name: tc-entry + type: nest + nested-attributes: tc-taprio-tc-entry-attrs + - + name: tc-taprio-sched-entry-list + attributes: + - + name: entry + type: nest + nested-attributes: tc-taprio-sched-entry + - + name: tc-taprio-sched-entry + attributes: + - + name: index + type: u32 + - + name: cmd + type: u8 + - + name: gate-mask + type: u32 + - + name: interval + type: u32 + - + name: tc-taprio-tc-entry-attrs + attributes: + - + name: index + type: u32 + - + name: max-sdu + type: u32 + - + name: fp + type: u32 + - + name: tc-tbf-attrs + attributes: + - + name: parms + type: binary + struct: tc-tbf-qopt + - + name: rtab + type: binary + - + name: ptab + type: binary + - + name: rate64 + type: u64 + - + name: prate4 + type: u64 + - + name: burst + type: u32 + - + name: pburst + type: u32 + - + name: pad + type: pad + - + name: tca-gact-attrs + attributes: + - + name: tm + type: binary + - + name: parms + type: binary + - + name: prob + type: binary + - + name: pad + type: pad + - + name: tca-stab-attrs + attributes: + - + name: base + type: binary + struct: tc-sizespec + - + name: data + type: binary + - + name: tca-stats-attrs + attributes: + - + name: basic + type: binary + - + name: rate-est + type: binary + - + name: queue + type: binary + - + name: app + type: binary # TODO sub-message needs 2+ level deep lookup + sub-message: tca-stats-app-msg + selector: kind + - + name: rate-est64 + type: binary + - + name: pad + type: pad + - + name: basic-hw + type: binary + - + name: pkt64 + type: binary + +sub-messages: + - + name: tc-options-msg + formats: + - + value: bfifo + fixed-header: tc-fifo-qopt + - + value: cake + attribute-set: tc-cake-attrs + - + value: cbs + attribute-set: tc-cbs-attrs + - + value: choke + attribute-set: tc-choke-attrs + - + value: clsact # no content + - + value: codel + attribute-set: tc-codel-attrs + - + value: drr + attribute-set: tc-drr-attrs + - + value: etf + attribute-set: tc-etf-attrs + - + value: ets + attribute-set: tc-ets-attrs + - + value: fq + attribute-set: tc-fq-attrs + - + value: fq_codel + attribute-set: tc-fq-codel-attrs + - + value: fq_pie + attribute-set: tc-fq-pie-attrs + - + value: flower + attribute-set: tc-flower-attrs + - + value: gred + attribute-set: tc-gred-attrs + - + value: hfsc + fixed-header: tc-hfsc-qopt + - + value: hhf + attribute-set: tc-hhf-attrs + - + value: htb + attribute-set: tc-htb-attrs + - + value: ingress # no content + - + value: mq # no content + - + value: mqprio + fixed-header: tc-mqprio-qopt + - + value: multiq + fixed-header: tc-multiq-qopt + - + value: netem + fixed-header: tc-netem-qopt + attribute-set: tc-netem-attrs + - + value: pfifo + fixed-header: tc-fifo-qopt + - + value: pfifo_fast + fixed-header: tc-prio-qopt + - + value: pfifo_head_drop + fixed-header: tc-fifo-qopt + - + value: pie + attribute-set: tc-pie-attrs + - + value: plug + fixed-header: tc-plug-qopt + - + value: prio + fixed-header: tc-prio-qopt + - + value: qfq + attribute-set: tc-qfq-attrs + - + value: red + attribute-set: tc-red-attrs + - + value: sfb + fixed-header: tc-sfb-qopt + - + value: sfq + fixed-header: tc-sfq-qopt-v1 + - + value: taprio + attribute-set: tc-taprio-attrs + - + value: tbf + attribute-set: tc-tbf-attrs + - + name: tc-act-options-msg + formats: + - + value: gact + attribute-set: tca-gact-attrs + - + name: tca-stats-app-msg + formats: + - + value: bfifo + - + value: blackhole + - + value: cake + attribute-set: tc-cake-stats-attrs + - + value: cbs + - + value: choke + - + value: clsact + - + value: codel + - + value: drr + - + value: etf + - + value: ets + - + value: fq + - + value: fq_codel + - + value: fq_pie + - + value: flower + - + value: gred + - + value: hfsc + - + value: hhf + - + value: htb + - + value: ingress + - + value: mq + - + value: mqprio + - + value: multiq + - + value: netem + - + value: noqueue + - + value: pfifo + - + value: pfifo_fast + - + value: pfifo_head_drop + - + value: pie + - + value: plug + - + value: prio + - + value: qfq + - + value: red + - + value: sfb + - + value: sfq + - + value: taprio + - + value: tbf + +operations: + enum-model: directional + list: + - + name: newqdisc + doc: Create new tc qdisc. + attribute-set: tc-attrs + fixed-header: tcmsg + do: + request: + value: 36 + attributes: &create-params + - kind + - options + - rate + - chain + - ingress-block + - egress-block + - + name: delqdisc + doc: Delete existing tc qdisc. + attribute-set: tc-attrs + fixed-header: tcmsg + do: + request: + value: 37 + - + name: getqdisc + doc: Get / dump tc qdisc information. + attribute-set: tc-attrs + fixed-header: tcmsg + do: + request: + value: 38 + attributes: + - dump-invisible + reply: + value: 36 + attributes: &tc-all + - kind + - options + - stats + - xstats + - rate + - fcnt + - stats2 + - stab + - chain + - ingress-block + - egress-block + - + name: newtclass + doc: Get / dump tc traffic class information. + attribute-set: tc-attrs + fixed-header: tcmsg + do: + request: + value: 40 + attributes: *create-params + - + name: deltclass + doc: Get / dump tc traffic class information. + attribute-set: tc-attrs + fixed-header: tcmsg + do: + request: + value: 41 + - + name: gettclass + doc: Get / dump tc traffic class information. + attribute-set: tc-attrs + fixed-header: tcmsg + do: + request: + value: 42 + reply: + value: 40 + attributes: *tc-all + - + name: newtfilter + doc: Get / dump tc filter information. + attribute-set: tc-attrs + fixed-header: tcmsg + do: + request: + value: 44 + attributes: *create-params + - + name: deltfilter + doc: Get / dump tc filter information. + attribute-set: tc-attrs + fixed-header: tcmsg + do: + request: + value: 45 + attributes: + - chain + - kind + - + name: gettfilter + doc: Get / dump tc filter information. + attribute-set: tc-attrs + fixed-header: tcmsg + do: + request: + value: 46 + attributes: + - chain + - kind + reply: + value: 44 + attributes: *tc-all + dump: + request: + value: 46 + attributes: + - chain + - dump-flags + reply: + value: 44 + attributes: *tc-all + - + name: newchain + doc: Get / dump tc chain information. + attribute-set: tc-attrs + fixed-header: tcmsg + do: + request: + value: 100 + attributes: *create-params + - + name: delchain + doc: Get / dump tc chain information. + attribute-set: tc-attrs + fixed-header: tcmsg + do: + request: + value: 101 + attributes: + - chain + - + name: getchain + doc: Get / dump tc chain information. + attribute-set: tc-attrs + fixed-header: tcmsg + do: + request: + value: 102 + attributes: + - chain + reply: + value: 100 + attributes: *tc-all + +mcast-groups: + list: + - + name: rtnlgrp-tc + value: 4 diff --git a/Documentation/networking/device_drivers/ethernet/intel/ice.rst b/Documentation/networking/device_drivers/ethernet/intel/ice.rst index e4d065c55ea8d1..5038e54586af66 100644 --- a/Documentation/networking/device_drivers/ethernet/intel/ice.rst +++ b/Documentation/networking/device_drivers/ethernet/intel/ice.rst @@ -895,6 +895,147 @@ driver writes raw bytes by the GNSS object to the receiver through i2c. Please refer to the hardware GNSS module documentation for configuration details. +Firmware (FW) logging +--------------------- +The driver supports FW logging via the debugfs interface on PF 0 only. The FW +running on the NIC must support FW logging; if the FW doesn't support FW logging +the 'fwlog' file will not get created in the ice debugfs directory. + +Module configuration +~~~~~~~~~~~~~~~~~~~~ +Firmware logging is configured on a per module basis. Each module can be set to +a value independent of the other modules (unless the module 'all' is specified). +The modules will be instantiated under the 'fwlog/modules' directory. + +The user can set the log level for a module by writing to the module file like +this:: + + # echo > /sys/kernel/debug/ice/0000\:18\:00.0/fwlog/modules/ + +where + +* log_level is a name as described below. Each level includes the + messages from the previous/lower level + + * none + * error + * warning + * normal + * verbose + +* module is a name that represents the module to receive events for. The + module names are + + * general + * ctrl + * link + * link_topo + * dnl + * i2c + * sdp + * mdio + * adminq + * hdma + * lldp + * dcbx + * dcb + * xlr + * nvm + * auth + * vpd + * iosf + * parser + * sw + * scheduler + * txq + * rsvd + * post + * watchdog + * task_dispatch + * mng + * synce + * health + * tsdrv + * pfreg + * mdlver + * all + +The name 'all' is special and allows the user to set all of the modules to the +specified log_level or to read the log_level of all of the modules. + +Example usage to configure the modules +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To set a single module to 'verbose':: + + # echo verbose > /sys/kernel/debug/ice/0000\:18\:00.0/fwlog/modules/link + +To set multiple modules then issue the command multiple times:: + + # echo verbose > /sys/kernel/debug/ice/0000\:18\:00.0/fwlog/modules/link + # echo warning > /sys/kernel/debug/ice/0000\:18\:00.0/fwlog/modules/ctrl + # echo none > /sys/kernel/debug/ice/0000\:18\:00.0/fwlog/modules/dcb + +To set all the modules to the same value:: + + # echo normal > /sys/kernel/debug/ice/0000\:18\:00.0/fwlog/modules/all + +To read the log_level of a specific module (e.g. module 'general'):: + + # cat /sys/kernel/debug/ice/0000\:18\:00.0/fwlog/modules/general + +To read the log_level of all the modules:: + + # cat /sys/kernel/debug/ice/0000\:18\:00.0/fwlog/modules/all + +Enabling FW log +~~~~~~~~~~~~~~~ +Configuring the modules indicates to the FW that the configured modules should +generate events that the driver is interested in, but it **does not** send the +events to the driver until the enable message is sent to the FW. To do this +the user can write a 1 (enable) or 0 (disable) to 'fwlog/enable'. An example +is:: + + # echo 1 > /sys/kernel/debug/ice/0000\:18\:00.0/fwlog/enable + +Retrieving FW log data +~~~~~~~~~~~~~~~~~~~~~~ +The FW log data can be retrieved by reading from 'fwlog/data'. The user can +write any value to 'fwlog/data' to clear the data. The data can only be cleared +when FW logging is disabled. The FW log data is a binary file that is sent to +Intel and used to help debug user issues. + +An example to read the data is:: + + # cat /sys/kernel/debug/ice/0000\:18\:00.0/fwlog/data > fwlog.bin + +An example to clear the data is:: + + # echo 0 > /sys/kernel/debug/ice/0000\:18\:00.0/fwlog/data + +Changing how often the log events are sent to the driver +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The driver receives FW log data from the Admin Receive Queue (ARQ). The +frequency that the FW sends the ARQ events can be configured by writing to +'fwlog/nr_messages'. The range is 1-128 (1 means push every log message, 128 +means push only when the max AQ command buffer is full). The suggested value is +10. The user can see what the value is configured to by reading +'fwlog/nr_messages'. An example to set the value is:: + + # echo 50 > /sys/kernel/debug/ice/0000\:18\:00.0/fwlog/nr_messages + +Configuring the amount of memory used to store FW log data +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The driver stores FW log data within the driver. The default size of the memory +used to store the data is 1MB. Some use cases may require more or less data so +the user can change the amount of memory that is allocated for FW log data. +To change the amount of memory then write to 'fwlog/log_size'. The value must be +one of: 128K, 256K, 512K, 1M, or 2M. FW logging must be disabled to change the +value. An example of changing the value is:: + + # echo 128K > /sys/kernel/debug/ice/0000\:18\:00.0/fwlog/log_size + + Performance Optimization ======================== Driver defaults are meant to fit a wide variety of workloads, but if further diff --git a/Documentation/networking/xdp-rx-metadata.rst b/Documentation/networking/xdp-rx-metadata.rst index e3e9420fd81742..a6e0ece18be54b 100644 --- a/Documentation/networking/xdp-rx-metadata.rst +++ b/Documentation/networking/xdp-rx-metadata.rst @@ -20,7 +20,13 @@ Currently, the following kfuncs are supported. In the future, as more metadata is supported, this set will grow: .. kernel-doc:: net/core/xdp.c - :identifiers: bpf_xdp_metadata_rx_timestamp bpf_xdp_metadata_rx_hash + :identifiers: bpf_xdp_metadata_rx_timestamp + +.. kernel-doc:: net/core/xdp.c + :identifiers: bpf_xdp_metadata_rx_hash + +.. kernel-doc:: net/core/xdp.c + :identifiers: bpf_xdp_metadata_rx_vlan_tag An XDP program can use these kfuncs to read the metadata into stack variables for its own consumption. Or, to pass the metadata on to other diff --git a/Documentation/networking/xsk-tx-metadata.rst b/Documentation/networking/xsk-tx-metadata.rst index 97ecfa480d00b4..bd033fe95cca5f 100644 --- a/Documentation/networking/xsk-tx-metadata.rst +++ b/Documentation/networking/xsk-tx-metadata.rst @@ -1,3 +1,5 @@ +.. SPDX-License-Identifier: GPL-2.0 + ================== AF_XDP TX Metadata ================== diff --git a/Documentation/userspace-api/netlink/netlink-raw.rst b/Documentation/userspace-api/netlink/netlink-raw.rst index f07fb9b9c101ea..1e14f5f22b8e0f 100644 --- a/Documentation/userspace-api/netlink/netlink-raw.rst +++ b/Documentation/userspace-api/netlink/netlink-raw.rst @@ -14,7 +14,8 @@ Specification The netlink-raw schema extends the :doc:`genetlink-legacy ` schema with properties that are needed to specify the protocol numbers and multicast IDs used by raw netlink families. See :ref:`classic_netlink` for more -information. +information. The raw netlink families also make use of type-specific +sub-messages. Globals ------- @@ -56,3 +57,96 @@ group registration. - name: rtnlgrp-mctp-ifaddr value: 34 + +Sub-messages +------------ + +Several raw netlink families such as +:doc:`rt_link<../../networking/netlink_spec/rt_link>` and +:doc:`tc<../../networking/netlink_spec/tc>` use attribute nesting as an +abstraction to carry module specific information. + +Conceptually it looks as follows:: + + [OUTER NEST OR MESSAGE LEVEL] + [GENERIC ATTR 1] + [GENERIC ATTR 2] + [GENERIC ATTR 3] + [GENERIC ATTR - wrapper] + [MODULE SPECIFIC ATTR 1] + [MODULE SPECIFIC ATTR 2] + +The ``GENERIC ATTRs`` at the outer level are defined in the core (or rt_link or +core TC), while specific drivers, TC classifiers, qdiscs etc. can carry their +own information wrapped in the ``GENERIC ATTR - wrapper``. Even though the +example above shows attributes nesting inside the wrapper, the modules generally +have full freedom to define the format of the nest. In practice the payload of +the wrapper attr has very similar characteristics to a netlink message. It may +contain a fixed header / structure, netlink attributes, or both. Because of +those shared characteristics we refer to the payload of the wrapper attribute as +a sub-message. + +A sub-message attribute uses the value of another attribute as a selector key to +choose the right sub-message format. For example if the following attribute has +already been decoded: + +.. code-block:: json + + { "kind": "gre" } + +and we encounter the following attribute spec: + +.. code-block:: yaml + + - + name: data + type: sub-message + sub-message: linkinfo-data-msg + selector: kind + +Then we look for a sub-message definition called ``linkinfo-data-msg`` and use +the value of the ``kind`` attribute i.e. ``gre`` as the key to choose the +correct format for the sub-message: + +.. code-block:: yaml + + sub-messages: + name: linkinfo-data-msg + formats: + - + value: bridge + attribute-set: linkinfo-bridge-attrs + - + value: gre + attribute-set: linkinfo-gre-attrs + - + value: geneve + attribute-set: linkinfo-geneve-attrs + +This would decode the attribute value as a sub-message with the attribute-set +called ``linkinfo-gre-attrs`` as the attribute space. + +A sub-message can have an optional ``fixed-header`` followed by zero or more +attributes from an ``attribute-set``. For example the following +``tc-options-msg`` sub-message defines message formats that use a mixture of +``fixed-header``, ``attribute-set`` or both together: + +.. code-block:: yaml + + sub-messages: + - + name: tc-options-msg + formats: + - + value: bfifo + fixed-header: tc-fifo-qopt + - + value: cake + attribute-set: tc-cake-attrs + - + value: netem + fixed-header: tc-netem-qopt + attribute-set: tc-netem-attrs + +Note that a selector attribute must appear in a netlink message before any +sub-message attributes that depend on it. diff --git a/MAINTAINERS b/MAINTAINERS index 49c1caad5d9930..dda78b4ce70767 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -3072,6 +3072,14 @@ S: Maintained F: Documentation/devicetree/bindings/net/asix,ax88796c.yaml F: drivers/net/ethernet/asix/ax88796c_* +ASIX PHY DRIVER [RUST] +M: FUJITA Tomonori +R: Trevor Gross +L: netdev@vger.kernel.org +L: rust-for-linux@vger.kernel.org +S: Maintained +F: drivers/net/phy/ax88796b_rust.rs + ASPEED CRYPTO DRIVER M: Neal Liu L: linux-aspeed@lists.ozlabs.org (moderated for non-subscribers) @@ -7873,6 +7881,14 @@ F: include/uapi/linux/mdio.h F: include/uapi/linux/mii.h F: net/core/of_net.c +ETHERNET PHY LIBRARY [RUST] +M: FUJITA Tomonori +R: Trevor Gross +L: netdev@vger.kernel.org +L: rust-for-linux@vger.kernel.org +S: Maintained +F: rust/kernel/net/phy.rs + EXEC & BINFMT API R: Eric Biederman R: Kees Cook @@ -15083,7 +15099,7 @@ W: https://github.com/multipath-tcp/mptcp_net-next/wiki B: https://github.com/multipath-tcp/mptcp_net-next/issues T: git https://github.com/multipath-tcp/mptcp_net-next.git export-net T: git https://github.com/multipath-tcp/mptcp_net-next.git export -F: Documentation/netlink/specs/mptcp.yaml +F: Documentation/netlink/specs/mptcp_pm.yaml F: Documentation/networking/mptcp-sysctl.rst F: include/net/mptcp.h F: include/trace/events/mptcp.h diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 7d4af64e398286..8955da5c47cf77 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -1828,7 +1828,7 @@ static void restore_args(struct jit_ctx *ctx, int args_off, int nregs) * */ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, - struct bpf_tramp_links *tlinks, void *orig_call, + struct bpf_tramp_links *tlinks, void *func_addr, int nregs, u32 flags) { int i; @@ -1926,7 +1926,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, if (flags & BPF_TRAMP_F_IP_ARG) { /* save ip address of the traced function */ - emit_addr_mov_i64(A64_R(10), (const u64)orig_call, ctx); + emit_addr_mov_i64(A64_R(10), (const u64)func_addr, ctx); emit(A64_STR64I(A64_R(10), A64_SP, ip_off), ctx); } @@ -2026,18 +2026,10 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, return ctx->idx; } -int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, - void *image_end, const struct btf_func_model *m, - u32 flags, struct bpf_tramp_links *tlinks, - void *orig_call) +static int btf_func_model_nregs(const struct btf_func_model *m) { - int i, ret; int nregs = m->nr_args; - int max_insns = ((long)image_end - (long)image) / AARCH64_INSN_SIZE; - struct jit_ctx ctx = { - .image = NULL, - .idx = 0, - }; + int i; /* extra registers needed for struct argument */ for (i = 0; i < MAX_BPF_FUNC_ARGS; i++) { @@ -2046,22 +2038,49 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, nregs += (m->arg_size[i] + 7) / 8 - 1; } + return nregs; +} + +int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, + struct bpf_tramp_links *tlinks, void *func_addr) +{ + struct jit_ctx ctx = { + .image = NULL, + .idx = 0, + }; + struct bpf_tramp_image im; + int nregs, ret; + + nregs = btf_func_model_nregs(m); /* the first 8 registers are used for arguments */ if (nregs > 8) return -ENOTSUPP; - ret = prepare_trampoline(&ctx, im, tlinks, orig_call, nregs, flags); + ret = prepare_trampoline(&ctx, &im, tlinks, func_addr, nregs, flags); if (ret < 0) return ret; - if (ret > max_insns) - return -EFBIG; + return ret < 0 ? ret : ret * AARCH64_INSN_SIZE; +} - ctx.image = image; - ctx.idx = 0; +int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, + void *image_end, const struct btf_func_model *m, + u32 flags, struct bpf_tramp_links *tlinks, + void *func_addr) +{ + int ret, nregs; + struct jit_ctx ctx = { + .image = image, + .idx = 0, + }; + + nregs = btf_func_model_nregs(m); + /* the first 8 registers are used for arguments */ + if (nregs > 8) + return -ENOTSUPP; jit_fill_hole(image, (unsigned int)(image_end - image)); - ret = prepare_trampoline(&ctx, im, tlinks, orig_call, nregs, flags); + ret = prepare_trampoline(&ctx, im, tlinks, func_addr, nregs, flags); if (ret > 0 && validate_code(&ctx) < 0) ret = -EINVAL; diff --git a/arch/riscv/include/asm/cfi.h b/arch/riscv/include/asm/cfi.h index 56bf9d69d5e38c..8f7a6225704499 100644 --- a/arch/riscv/include/asm/cfi.h +++ b/arch/riscv/include/asm/cfi.h @@ -7,8 +7,9 @@ * * Copyright (C) 2023 Google LLC */ +#include -#include +struct pt_regs; #ifdef CONFIG_CFI_CLANG enum bug_trap_type handle_cfi_failure(struct pt_regs *regs); diff --git a/arch/riscv/kernel/cfi.c b/arch/riscv/kernel/cfi.c index 820158d7a29130..6ec9dbd7292eec 100644 --- a/arch/riscv/kernel/cfi.c +++ b/arch/riscv/kernel/cfi.c @@ -4,7 +4,7 @@ * * Copyright (C) 2023 Google LLC */ -#include +#include #include /* diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index 8581693e62d396..58dc64dd94a82c 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -1029,23 +1029,28 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, return ret; } -int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, - void *image_end, const struct btf_func_model *m, - u32 flags, struct bpf_tramp_links *tlinks, - void *func_addr) +int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, + struct bpf_tramp_links *tlinks, void *func_addr) { - int ret; + struct bpf_tramp_image im; struct rv_jit_context ctx; + int ret; ctx.ninsns = 0; ctx.insns = NULL; ctx.ro_insns = NULL; - ret = __arch_prepare_bpf_trampoline(im, m, tlinks, func_addr, flags, &ctx); - if (ret < 0) - return ret; + ret = __arch_prepare_bpf_trampoline(&im, m, tlinks, func_addr, flags, &ctx); - if (ninsns_rvoff(ret) > (long)image_end - (long)image) - return -EFBIG; + return ret < 0 ? ret : ninsns_rvoff(ctx.ninsns); +} + +int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, + void *image_end, const struct btf_func_model *m, + u32 flags, struct bpf_tramp_links *tlinks, + void *func_addr) +{ + int ret; + struct rv_jit_context ctx; ctx.ninsns = 0; /* diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index bf06b7283f0ca6..7f0a7b97ef4ce7 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -2362,7 +2362,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, return -ENOTSUPP; /* Return to %r14, since func_addr and %r0 are not available. */ - if (!func_addr && !(flags & BPF_TRAMP_F_ORIG_STACK)) + if ((!func_addr && !(flags & BPF_TRAMP_F_ORIG_STACK)) || + (flags & BPF_TRAMP_F_INDIRECT)) flags |= BPF_TRAMP_F_SKIP_FRAME; /* @@ -2637,6 +2638,21 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, return 0; } +int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, + struct bpf_tramp_links *tlinks, void *orig_call) +{ + struct bpf_tramp_image im; + struct bpf_tramp_jit tjit; + int ret; + + memset(&tjit, 0, sizeof(tjit)); + + ret = __arch_prepare_bpf_trampoline(&im, &tjit, m, flags, + tlinks, orig_call); + + return ret < 0 ? ret : tjit.common.prg; +} + int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, const struct btf_func_model *m, u32 flags, struct bpf_tramp_links *tlinks, @@ -2644,30 +2660,27 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, { struct bpf_tramp_jit tjit; int ret; - int i; - for (i = 0; i < 2; i++) { - if (i == 0) { - /* Compute offsets, check whether the code fits. */ - memset(&tjit, 0, sizeof(tjit)); - } else { - /* Generate the code. */ - tjit.common.prg = 0; - tjit.common.prg_buf = image; - } - ret = __arch_prepare_bpf_trampoline(im, &tjit, m, flags, - tlinks, func_addr); - if (ret < 0) - return ret; - if (tjit.common.prg > (char *)image_end - (char *)image) - /* - * Use the same error code as for exceeding - * BPF_MAX_TRAMP_LINKS. - */ - return -E2BIG; - } + /* Compute offsets, check whether the code fits. */ + memset(&tjit, 0, sizeof(tjit)); + ret = __arch_prepare_bpf_trampoline(im, &tjit, m, flags, + tlinks, func_addr); + + if (ret < 0) + return ret; + if (tjit.common.prg > (char *)image_end - (char *)image) + /* + * Use the same error code as for exceeding + * BPF_MAX_TRAMP_LINKS. + */ + return -E2BIG; + + tjit.common.prg = 0; + tjit.common.prg_buf = image; + ret = __arch_prepare_bpf_trampoline(im, &tjit, m, flags, + tlinks, func_addr); - return tjit.common.prg; + return ret < 0 ? ret : tjit.common.prg; } bool bpf_jit_supports_subprog_tailcalls(void) diff --git a/arch/x86/include/asm/cfi.h b/arch/x86/include/asm/cfi.h index 58dacd90daefc4..7cd7525579051d 100644 --- a/arch/x86/include/asm/cfi.h +++ b/arch/x86/include/asm/cfi.h @@ -7,16 +7,140 @@ * * Copyright (C) 2022 Google LLC */ +#include +#include -#include +/* + * An overview of the various calling conventions... + * + * Traditional: + * + * foo: + * ... code here ... + * ret + * + * direct caller: + * call foo + * + * indirect caller: + * lea foo(%rip), %r11 + * ... + * call *%r11 + * + * + * IBT: + * + * foo: + * endbr64 + * ... code here ... + * ret + * + * direct caller: + * call foo / call foo+4 + * + * indirect caller: + * lea foo(%rip), %r11 + * ... + * call *%r11 + * + * + * kCFI: + * + * __cfi_foo: + * movl $0x12345678, %eax + * # 11 nops when CONFIG_CALL_PADDING + * foo: + * endbr64 # when IBT + * ... code here ... + * ret + * + * direct call: + * call foo # / call foo+4 when IBT + * + * indirect call: + * lea foo(%rip), %r11 + * ... + * movl $(-0x12345678), %r10d + * addl -4(%r11), %r10d # -15 when CONFIG_CALL_PADDING + * jz 1f + * ud2 + * 1:call *%r11 + * + * + * FineIBT (builds as kCFI + CALL_PADDING + IBT + RETPOLINE and runtime patches into): + * + * __cfi_foo: + * endbr64 + * subl 0x12345678, %r10d + * jz foo + * ud2 + * nop + * foo: + * osp nop3 # was endbr64 + * ... code here ... + * ret + * + * direct caller: + * call foo / call foo+4 + * + * indirect caller: + * lea foo(%rip), %r11 + * ... + * movl $0x12345678, %r10d + * subl $16, %r11 + * nop4 + * call *%r11 + * + */ +enum cfi_mode { + CFI_DEFAULT, /* FineIBT if hardware has IBT, otherwise kCFI */ + CFI_OFF, /* Taditional / IBT depending on .config */ + CFI_KCFI, /* Optionally CALL_PADDING, IBT, RETPOLINE */ + CFI_FINEIBT, /* see arch/x86/kernel/alternative.c */ +}; + +extern enum cfi_mode cfi_mode; + +struct pt_regs; #ifdef CONFIG_CFI_CLANG enum bug_trap_type handle_cfi_failure(struct pt_regs *regs); +#define __bpfcall +extern u32 cfi_bpf_hash; +extern u32 cfi_bpf_subprog_hash; + +static inline int cfi_get_offset(void) +{ + switch (cfi_mode) { + case CFI_FINEIBT: + return 16; + case CFI_KCFI: + if (IS_ENABLED(CONFIG_CALL_PADDING)) + return 16; + return 5; + default: + return 0; + } +} +#define cfi_get_offset cfi_get_offset + +extern u32 cfi_get_func_hash(void *func); + #else static inline enum bug_trap_type handle_cfi_failure(struct pt_regs *regs) { return BUG_TRAP_TYPE_NONE; } +#define cfi_bpf_hash 0U +#define cfi_bpf_subprog_hash 0U +static inline u32 cfi_get_func_hash(void *func) +{ + return 0; +} #endif /* CONFIG_CFI_CLANG */ +#if HAS_KERNEL_IBT == 1 +#define CFI_NOSEAL(x) asm(IBT_NOSEAL(__stringify(x))) +#endif + #endif /* _ASM_X86_CFI_H */ diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 73be3931e4f060..49c2a62ba5e4e9 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -30,6 +30,7 @@ #include #include #include +#include int __read_mostly alternatives_patched; @@ -832,15 +833,82 @@ void __init_or_module apply_seal_endbr(s32 *start, s32 *end) { } #endif /* CONFIG_X86_KERNEL_IBT */ #ifdef CONFIG_FINEIBT +#define __CFI_DEFAULT CFI_DEFAULT +#elif defined(CONFIG_CFI_CLANG) +#define __CFI_DEFAULT CFI_KCFI +#else +#define __CFI_DEFAULT CFI_OFF +#endif -enum cfi_mode { - CFI_DEFAULT, - CFI_OFF, - CFI_KCFI, - CFI_FINEIBT, -}; +enum cfi_mode cfi_mode __ro_after_init = __CFI_DEFAULT; + +#ifdef CONFIG_CFI_CLANG +struct bpf_insn; + +/* Must match bpf_func_t / DEFINE_BPF_PROG_RUN() */ +extern unsigned int __bpf_prog_runX(const void *ctx, + const struct bpf_insn *insn); + +/* + * Force a reference to the external symbol so the compiler generates + * __kcfi_typid. + */ +__ADDRESSABLE(__bpf_prog_runX); + +/* u32 __ro_after_init cfi_bpf_hash = __kcfi_typeid___bpf_prog_runX; */ +asm ( +" .pushsection .data..ro_after_init,\"aw\",@progbits \n" +" .type cfi_bpf_hash,@object \n" +" .globl cfi_bpf_hash \n" +" .p2align 2, 0x0 \n" +"cfi_bpf_hash: \n" +" .long __kcfi_typeid___bpf_prog_runX \n" +" .size cfi_bpf_hash, 4 \n" +" .popsection \n" +); + +/* Must match bpf_callback_t */ +extern u64 __bpf_callback_fn(u64, u64, u64, u64, u64); + +__ADDRESSABLE(__bpf_callback_fn); + +/* u32 __ro_after_init cfi_bpf_subprog_hash = __kcfi_typeid___bpf_callback_fn; */ +asm ( +" .pushsection .data..ro_after_init,\"aw\",@progbits \n" +" .type cfi_bpf_subprog_hash,@object \n" +" .globl cfi_bpf_subprog_hash \n" +" .p2align 2, 0x0 \n" +"cfi_bpf_subprog_hash: \n" +" .long __kcfi_typeid___bpf_callback_fn \n" +" .size cfi_bpf_subprog_hash, 4 \n" +" .popsection \n" +); + +u32 cfi_get_func_hash(void *func) +{ + u32 hash; + + func -= cfi_get_offset(); + switch (cfi_mode) { + case CFI_FINEIBT: + func += 7; + break; + case CFI_KCFI: + func += 1; + break; + default: + return 0; + } + + if (get_kernel_nofault(hash, func)) + return 0; + + return hash; +} +#endif + +#ifdef CONFIG_FINEIBT -static enum cfi_mode cfi_mode __ro_after_init = CFI_DEFAULT; static bool cfi_rand __ro_after_init = true; static u32 cfi_seed __ro_after_init; @@ -1149,8 +1217,11 @@ static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, goto err; if (cfi_rand) { - if (builtin) + if (builtin) { cfi_seed = get_random_u32(); + cfi_bpf_hash = cfi_rehash(cfi_bpf_hash); + cfi_bpf_subprog_hash = cfi_rehash(cfi_bpf_subprog_hash); + } ret = cfi_rand_preamble(start_cfi, end_cfi); if (ret) diff --git a/arch/x86/kernel/cfi.c b/arch/x86/kernel/cfi.c index 8674a5c0c031d3..e6bf78fac14622 100644 --- a/arch/x86/kernel/cfi.c +++ b/arch/x86/kernel/cfi.c @@ -4,10 +4,10 @@ * * Copyright (C) 2022 Google LLC */ -#include +#include +#include #include #include -#include /* * Returns the target address and the expected type when regs->ip points diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index e89e415aa74353..bdacbb84456d97 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -17,6 +17,7 @@ #include #include #include +#include static bool all_callee_regs_used[4] = {true, true, true, true}; @@ -51,9 +52,11 @@ static u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len) do { EMIT4(b1, b2, b3, b4); EMIT(off, 4); } while (0) #ifdef CONFIG_X86_KERNEL_IBT -#define EMIT_ENDBR() EMIT(gen_endbr(), 4) +#define EMIT_ENDBR() EMIT(gen_endbr(), 4) +#define EMIT_ENDBR_POISON() EMIT(gen_endbr_poison(), 4) #else #define EMIT_ENDBR() +#define EMIT_ENDBR_POISON() #endif static bool is_imm8(int value) @@ -304,6 +307,69 @@ static void pop_callee_regs(u8 **pprog, bool *callee_regs_used) *pprog = prog; } +/* + * Emit the various CFI preambles, see asm/cfi.h and the comments about FineIBT + * in arch/x86/kernel/alternative.c + */ + +static void emit_fineibt(u8 **pprog, u32 hash) +{ + u8 *prog = *pprog; + + EMIT_ENDBR(); + EMIT3_off32(0x41, 0x81, 0xea, hash); /* subl $hash, %r10d */ + EMIT2(0x74, 0x07); /* jz.d8 +7 */ + EMIT2(0x0f, 0x0b); /* ud2 */ + EMIT1(0x90); /* nop */ + EMIT_ENDBR_POISON(); + + *pprog = prog; +} + +static void emit_kcfi(u8 **pprog, u32 hash) +{ + u8 *prog = *pprog; + + EMIT1_off32(0xb8, hash); /* movl $hash, %eax */ +#ifdef CONFIG_CALL_PADDING + EMIT1(0x90); + EMIT1(0x90); + EMIT1(0x90); + EMIT1(0x90); + EMIT1(0x90); + EMIT1(0x90); + EMIT1(0x90); + EMIT1(0x90); + EMIT1(0x90); + EMIT1(0x90); + EMIT1(0x90); +#endif + EMIT_ENDBR(); + + *pprog = prog; +} + +static void emit_cfi(u8 **pprog, u32 hash) +{ + u8 *prog = *pprog; + + switch (cfi_mode) { + case CFI_FINEIBT: + emit_fineibt(&prog, hash); + break; + + case CFI_KCFI: + emit_kcfi(&prog, hash); + break; + + default: + EMIT_ENDBR(); + break; + } + + *pprog = prog; +} + /* * Emit x86-64 prologue code for BPF program. * bpf_tail_call helper will skip the first X86_TAIL_CALL_OFFSET bytes @@ -315,10 +381,10 @@ static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf, { u8 *prog = *pprog; + emit_cfi(&prog, is_subprog ? cfi_bpf_subprog_hash : cfi_bpf_hash); /* BPF trampoline can be made to work without these nops, * but let's waste 5 bytes for now and optimize later */ - EMIT_ENDBR(); memcpy(prog, x86_nops[5], X86_PATCH_SIZE); prog += X86_PATCH_SIZE; if (!ebpf_from_cbpf) { @@ -2198,7 +2264,8 @@ static void restore_regs(const struct btf_func_model *m, u8 **prog, static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, struct bpf_tramp_link *l, int stack_size, - int run_ctx_off, bool save_ret) + int run_ctx_off, bool save_ret, + void *image, void *rw_image) { u8 *prog = *pprog; u8 *jmp_insn; @@ -2226,7 +2293,7 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, else EMIT4(0x48, 0x8D, 0x75, -run_ctx_off); - if (emit_rsb_call(&prog, bpf_trampoline_enter(p), prog)) + if (emit_rsb_call(&prog, bpf_trampoline_enter(p), image + (prog - (u8 *)rw_image))) return -EINVAL; /* remember prog start time returned by __bpf_prog_enter */ emit_mov_reg(&prog, true, BPF_REG_6, BPF_REG_0); @@ -2250,7 +2317,7 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, (long) p->insnsi >> 32, (u32) (long) p->insnsi); /* call JITed bpf program or interpreter */ - if (emit_rsb_call(&prog, p->bpf_func, prog)) + if (emit_rsb_call(&prog, p->bpf_func, image + (prog - (u8 *)rw_image))) return -EINVAL; /* @@ -2277,7 +2344,7 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, EMIT3_off32(0x48, 0x8D, 0x95, -run_ctx_off); else EMIT4(0x48, 0x8D, 0x55, -run_ctx_off); - if (emit_rsb_call(&prog, bpf_trampoline_exit(p), prog)) + if (emit_rsb_call(&prog, bpf_trampoline_exit(p), image + (prog - (u8 *)rw_image))) return -EINVAL; *pprog = prog; @@ -2312,14 +2379,15 @@ static int emit_cond_near_jump(u8 **pprog, void *func, void *ip, u8 jmp_cond) static int invoke_bpf(const struct btf_func_model *m, u8 **pprog, struct bpf_tramp_links *tl, int stack_size, - int run_ctx_off, bool save_ret) + int run_ctx_off, bool save_ret, + void *image, void *rw_image) { int i; u8 *prog = *pprog; for (i = 0; i < tl->nr_links; i++) { if (invoke_bpf_prog(m, &prog, tl->links[i], stack_size, - run_ctx_off, save_ret)) + run_ctx_off, save_ret, image, rw_image)) return -EINVAL; } *pprog = prog; @@ -2328,7 +2396,8 @@ static int invoke_bpf(const struct btf_func_model *m, u8 **pprog, static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog, struct bpf_tramp_links *tl, int stack_size, - int run_ctx_off, u8 **branches) + int run_ctx_off, u8 **branches, + void *image, void *rw_image) { u8 *prog = *pprog; int i; @@ -2339,7 +2408,8 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog, emit_mov_imm32(&prog, false, BPF_REG_0, 0); emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8); for (i = 0; i < tl->nr_links; i++) { - if (invoke_bpf_prog(m, &prog, tl->links[i], stack_size, run_ctx_off, true)) + if (invoke_bpf_prog(m, &prog, tl->links[i], stack_size, run_ctx_off, true, + image, rw_image)) return -EINVAL; /* mod_ret prog stored return value into [rbp - 8]. Emit: @@ -2422,10 +2492,11 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog, * add rsp, 8 // skip eth_type_trans's frame * ret // return to its caller */ -int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, - const struct btf_func_model *m, u32 flags, - struct bpf_tramp_links *tlinks, - void *func_addr) +static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_image, + void *rw_image_end, void *image, + const struct btf_func_model *m, u32 flags, + struct bpf_tramp_links *tlinks, + void *func_addr) { int i, ret, nr_regs = m->nr_args, stack_size = 0; int regs_off, nregs_off, ip_off, run_ctx_off, arg_stack_off, rbx_off; @@ -2437,10 +2508,19 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i u8 *prog; bool save_ret; + /* + * F_INDIRECT is only compatible with F_RET_FENTRY_RET, it is + * explicitly incompatible with F_CALL_ORIG | F_SKIP_FRAME | F_IP_ARG + * because @func_addr. + */ + WARN_ON_ONCE((flags & BPF_TRAMP_F_INDIRECT) && + (flags & ~(BPF_TRAMP_F_INDIRECT | BPF_TRAMP_F_RET_FENTRY_RET))); + /* extra registers for struct arguments */ - for (i = 0; i < m->nr_args; i++) + for (i = 0; i < m->nr_args; i++) { if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG) nr_regs += (m->arg_size[i] + 7) / 8 - 1; + } /* x86-64 supports up to MAX_BPF_FUNC_ARGS arguments. 1-6 * are passed through regs, the remains are through stack. @@ -2521,22 +2601,29 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i orig_call += X86_PATCH_SIZE; } - prog = image; + prog = rw_image; - EMIT_ENDBR(); - /* - * This is the direct-call trampoline, as such it needs accounting - * for the __fentry__ call. - */ - x86_call_depth_emit_accounting(&prog, NULL); + if (flags & BPF_TRAMP_F_INDIRECT) { + /* + * Indirect call for bpf_struct_ops + */ + emit_cfi(&prog, cfi_get_func_hash(func_addr)); + } else { + /* + * Direct-call fentry stub, as such it needs accounting for the + * __fentry__ call. + */ + x86_call_depth_emit_accounting(&prog, NULL); + } EMIT1(0x55); /* push rbp */ EMIT3(0x48, 0x89, 0xE5); /* mov rbp, rsp */ - if (!is_imm8(stack_size)) + if (!is_imm8(stack_size)) { /* sub rsp, stack_size */ EMIT3_off32(0x48, 0x81, 0xEC, stack_size); - else + } else { /* sub rsp, stack_size */ EMIT4(0x48, 0x83, 0xEC, stack_size); + } if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) EMIT1(0x50); /* push rax */ /* mov QWORD PTR [rbp - rbx_off], rbx */ @@ -2563,16 +2650,18 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i if (flags & BPF_TRAMP_F_CALL_ORIG) { /* arg1: mov rdi, im */ emit_mov_imm64(&prog, BPF_REG_1, (long) im >> 32, (u32) (long) im); - if (emit_rsb_call(&prog, __bpf_tramp_enter, prog)) { + if (emit_rsb_call(&prog, __bpf_tramp_enter, + image + (prog - (u8 *)rw_image))) { ret = -EINVAL; goto cleanup; } } - if (fentry->nr_links) + if (fentry->nr_links) { if (invoke_bpf(m, &prog, fentry, regs_off, run_ctx_off, - flags & BPF_TRAMP_F_RET_FENTRY_RET)) + flags & BPF_TRAMP_F_RET_FENTRY_RET, image, rw_image)) return -EINVAL; + } if (fmod_ret->nr_links) { branches = kcalloc(fmod_ret->nr_links, sizeof(u8 *), @@ -2581,7 +2670,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i return -ENOMEM; if (invoke_bpf_mod_ret(m, &prog, fmod_ret, regs_off, - run_ctx_off, branches)) { + run_ctx_off, branches, image, rw_image)) { ret = -EINVAL; goto cleanup; } @@ -2591,25 +2680,26 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i restore_regs(m, &prog, regs_off); save_args(m, &prog, arg_stack_off, true); - if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) + if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) { /* Before calling the original function, restore the * tail_call_cnt from stack to rax. */ RESTORE_TAIL_CALL_CNT(stack_size); + } if (flags & BPF_TRAMP_F_ORIG_STACK) { emit_ldx(&prog, BPF_DW, BPF_REG_6, BPF_REG_FP, 8); EMIT2(0xff, 0xd3); /* call *rbx */ } else { /* call original function */ - if (emit_rsb_call(&prog, orig_call, prog)) { + if (emit_rsb_call(&prog, orig_call, image + (prog - (u8 *)rw_image))) { ret = -EINVAL; goto cleanup; } } /* remember return value in a stack for bpf prog to access */ emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8); - im->ip_after_call = prog; + im->ip_after_call = image + (prog - (u8 *)rw_image); memcpy(prog, x86_nops[5], X86_PATCH_SIZE); prog += X86_PATCH_SIZE; } @@ -2624,16 +2714,19 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i /* Update the branches saved in invoke_bpf_mod_ret with the * aligned address of do_fexit. */ - for (i = 0; i < fmod_ret->nr_links; i++) - emit_cond_near_jump(&branches[i], prog, branches[i], - X86_JNE); + for (i = 0; i < fmod_ret->nr_links; i++) { + emit_cond_near_jump(&branches[i], image + (prog - (u8 *)rw_image), + image + (branches[i] - (u8 *)rw_image), X86_JNE); + } } - if (fexit->nr_links) - if (invoke_bpf(m, &prog, fexit, regs_off, run_ctx_off, false)) { + if (fexit->nr_links) { + if (invoke_bpf(m, &prog, fexit, regs_off, run_ctx_off, + false, image, rw_image)) { ret = -EINVAL; goto cleanup; } + } if (flags & BPF_TRAMP_F_RESTORE_REGS) restore_regs(m, &prog, regs_off); @@ -2643,18 +2736,19 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i * restored to R0. */ if (flags & BPF_TRAMP_F_CALL_ORIG) { - im->ip_epilogue = prog; + im->ip_epilogue = image + (prog - (u8 *)rw_image); /* arg1: mov rdi, im */ emit_mov_imm64(&prog, BPF_REG_1, (long) im >> 32, (u32) (long) im); - if (emit_rsb_call(&prog, __bpf_tramp_exit, prog)) { + if (emit_rsb_call(&prog, __bpf_tramp_exit, image + (prog - (u8 *)rw_image))) { ret = -EINVAL; goto cleanup; } - } else if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) + } else if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) { /* Before running the original function, restore the * tail_call_cnt from stack to rax. */ RESTORE_TAIL_CALL_CNT(stack_size); + } /* restore return value of orig_call or fentry prog back into RAX */ if (save_ret) @@ -2662,22 +2756,94 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i emit_ldx(&prog, BPF_DW, BPF_REG_6, BPF_REG_FP, -rbx_off); EMIT1(0xC9); /* leave */ - if (flags & BPF_TRAMP_F_SKIP_FRAME) + if (flags & BPF_TRAMP_F_SKIP_FRAME) { /* skip our return address and return to parent */ EMIT4(0x48, 0x83, 0xC4, 8); /* add rsp, 8 */ - emit_return(&prog, prog); + } + emit_return(&prog, image + (prog - (u8 *)rw_image)); /* Make sure the trampoline generation logic doesn't overflow */ - if (WARN_ON_ONCE(prog > (u8 *)image_end - BPF_INSN_SAFETY)) { + if (WARN_ON_ONCE(prog > (u8 *)rw_image_end - BPF_INSN_SAFETY)) { ret = -EFAULT; goto cleanup; } - ret = prog - (u8 *)image; + ret = prog - (u8 *)rw_image + BPF_INSN_SAFETY; cleanup: kfree(branches); return ret; } +void *arch_alloc_bpf_trampoline(unsigned int size) +{ + return bpf_prog_pack_alloc(size, jit_fill_hole); +} + +void arch_free_bpf_trampoline(void *image, unsigned int size) +{ + bpf_prog_pack_free(image, size); +} + +void arch_protect_bpf_trampoline(void *image, unsigned int size) +{ +} + +void arch_unprotect_bpf_trampoline(void *image, unsigned int size) +{ +} + +int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, + const struct btf_func_model *m, u32 flags, + struct bpf_tramp_links *tlinks, + void *func_addr) +{ + void *rw_image, *tmp; + int ret; + u32 size = image_end - image; + + /* rw_image doesn't need to be in module memory range, so we can + * use kvmalloc. + */ + rw_image = kvmalloc(size, GFP_KERNEL); + if (!rw_image) + return -ENOMEM; + + ret = __arch_prepare_bpf_trampoline(im, rw_image, rw_image + size, image, m, + flags, tlinks, func_addr); + if (ret < 0) + goto out; + + tmp = bpf_arch_text_copy(image, rw_image, size); + if (IS_ERR(tmp)) + ret = PTR_ERR(tmp); +out: + kvfree(rw_image); + return ret; +} + +int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, + struct bpf_tramp_links *tlinks, void *func_addr) +{ + struct bpf_tramp_image im; + void *image; + int ret; + + /* Allocate a temporary buffer for __arch_prepare_bpf_trampoline(). + * This will NOT cause fragmentation in direct map, as we do not + * call set_memory_*() on this buffer. + * + * We cannot use kvmalloc here, because we need image to be in + * module memory range. + */ + image = bpf_jit_alloc_exec(PAGE_SIZE); + if (!image) + return -ENOMEM; + + ret = __arch_prepare_bpf_trampoline(&im, image, image + PAGE_SIZE, image, + m, flags, tlinks, func_addr); + bpf_jit_free_exec(image); + return ret; +} + static int emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs, u8 *image, u8 *buf) { u8 *jg_reloc, *prog = *pprog; @@ -2935,9 +3101,16 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) jit_data->header = header; jit_data->rw_header = rw_header; } - prog->bpf_func = (void *)image; + /* + * ctx.prog_offset is used when CFI preambles put code *before* + * the function. See emit_cfi(). For FineIBT specifically this code + * can also be executed and bpf_prog_kallsyms_add() will + * generate an additional symbol to cover this, hence also + * decrement proglen. + */ + prog->bpf_func = (void *)image + cfi_get_offset(); prog->jited = 1; - prog->jited_len = proglen; + prog->jited_len = proglen - cfi_get_offset(); } else { prog = orig_prog; } @@ -2992,6 +3165,7 @@ void bpf_jit_free(struct bpf_prog *prog) kvfree(jit_data->addrs); kfree(jit_data); } + prog->bpf_func = (void *)prog->bpf_func - cfi_get_offset(); hdr = bpf_jit_binary_pack_hdr(prog); bpf_jit_binary_pack_free(hdr, NULL); WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(prog)); diff --git a/drivers/bcma/driver_pci_host.c b/drivers/bcma/driver_pci_host.c index aa0581cda71855..ed3be52ab63d16 100644 --- a/drivers/bcma/driver_pci_host.c +++ b/drivers/bcma/driver_pci_host.c @@ -280,7 +280,7 @@ static u8 bcma_find_pci_capability(struct bcma_drv_pci *pc, unsigned int dev, /* check for Header type 0 */ bcma_extpci_read_config(pc, dev, func, PCI_HEADER_TYPE, &byte_val, sizeof(u8)); - if ((byte_val & 0x7F) != PCI_HEADER_TYPE_NORMAL) + if ((byte_val & PCI_HEADER_TYPE_MASK) != PCI_HEADER_TYPE_NORMAL) return cap_ptr; /* check if the capability pointer field exists */ diff --git a/drivers/media/rc/bpf-lirc.c b/drivers/media/rc/bpf-lirc.c index fe17c7f98e8101..6d07693c6b9f5d 100644 --- a/drivers/media/rc/bpf-lirc.c +++ b/drivers/media/rc/bpf-lirc.c @@ -110,7 +110,7 @@ lirc_mode2_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_get_prandom_u32: return &bpf_get_prandom_u32_proto; case BPF_FUNC_trace_printk: - if (perfmon_capable()) + if (bpf_token_capable(prog->aux->token, CAP_PERFMON)) return bpf_get_trace_printk_proto(); fallthrough; default: diff --git a/drivers/net/dsa/mv88e6xxx/chip.c b/drivers/net/dsa/mv88e6xxx/chip.c index 07a22c74fe810a..383b3c4d6f599c 100644 --- a/drivers/net/dsa/mv88e6xxx/chip.c +++ b/drivers/net/dsa/mv88e6xxx/chip.c @@ -943,76 +943,94 @@ static void mv88e6xxx_mac_link_up(struct dsa_switch *ds, int port, static int mv88e6xxx_stats_snapshot(struct mv88e6xxx_chip *chip, int port) { + int err; + if (!chip->info->ops->stats_snapshot) return -EOPNOTSUPP; - return chip->info->ops->stats_snapshot(chip, port); -} - -static struct mv88e6xxx_hw_stat mv88e6xxx_hw_stats[] = { - { "in_good_octets", 8, 0x00, STATS_TYPE_BANK0, }, - { "in_bad_octets", 4, 0x02, STATS_TYPE_BANK0, }, - { "in_unicast", 4, 0x04, STATS_TYPE_BANK0, }, - { "in_broadcasts", 4, 0x06, STATS_TYPE_BANK0, }, - { "in_multicasts", 4, 0x07, STATS_TYPE_BANK0, }, - { "in_pause", 4, 0x16, STATS_TYPE_BANK0, }, - { "in_undersize", 4, 0x18, STATS_TYPE_BANK0, }, - { "in_fragments", 4, 0x19, STATS_TYPE_BANK0, }, - { "in_oversize", 4, 0x1a, STATS_TYPE_BANK0, }, - { "in_jabber", 4, 0x1b, STATS_TYPE_BANK0, }, - { "in_rx_error", 4, 0x1c, STATS_TYPE_BANK0, }, - { "in_fcs_error", 4, 0x1d, STATS_TYPE_BANK0, }, - { "out_octets", 8, 0x0e, STATS_TYPE_BANK0, }, - { "out_unicast", 4, 0x10, STATS_TYPE_BANK0, }, - { "out_broadcasts", 4, 0x13, STATS_TYPE_BANK0, }, - { "out_multicasts", 4, 0x12, STATS_TYPE_BANK0, }, - { "out_pause", 4, 0x15, STATS_TYPE_BANK0, }, - { "excessive", 4, 0x11, STATS_TYPE_BANK0, }, - { "collisions", 4, 0x1e, STATS_TYPE_BANK0, }, - { "deferred", 4, 0x05, STATS_TYPE_BANK0, }, - { "single", 4, 0x14, STATS_TYPE_BANK0, }, - { "multiple", 4, 0x17, STATS_TYPE_BANK0, }, - { "out_fcs_error", 4, 0x03, STATS_TYPE_BANK0, }, - { "late", 4, 0x1f, STATS_TYPE_BANK0, }, - { "hist_64bytes", 4, 0x08, STATS_TYPE_BANK0, }, - { "hist_65_127bytes", 4, 0x09, STATS_TYPE_BANK0, }, - { "hist_128_255bytes", 4, 0x0a, STATS_TYPE_BANK0, }, - { "hist_256_511bytes", 4, 0x0b, STATS_TYPE_BANK0, }, - { "hist_512_1023bytes", 4, 0x0c, STATS_TYPE_BANK0, }, - { "hist_1024_max_bytes", 4, 0x0d, STATS_TYPE_BANK0, }, - { "sw_in_discards", 4, 0x10, STATS_TYPE_PORT, }, - { "sw_in_filtered", 2, 0x12, STATS_TYPE_PORT, }, - { "sw_out_filtered", 2, 0x13, STATS_TYPE_PORT, }, - { "in_discards", 4, 0x00, STATS_TYPE_BANK1, }, - { "in_filtered", 4, 0x01, STATS_TYPE_BANK1, }, - { "in_accepted", 4, 0x02, STATS_TYPE_BANK1, }, - { "in_bad_accepted", 4, 0x03, STATS_TYPE_BANK1, }, - { "in_good_avb_class_a", 4, 0x04, STATS_TYPE_BANK1, }, - { "in_good_avb_class_b", 4, 0x05, STATS_TYPE_BANK1, }, - { "in_bad_avb_class_a", 4, 0x06, STATS_TYPE_BANK1, }, - { "in_bad_avb_class_b", 4, 0x07, STATS_TYPE_BANK1, }, - { "tcam_counter_0", 4, 0x08, STATS_TYPE_BANK1, }, - { "tcam_counter_1", 4, 0x09, STATS_TYPE_BANK1, }, - { "tcam_counter_2", 4, 0x0a, STATS_TYPE_BANK1, }, - { "tcam_counter_3", 4, 0x0b, STATS_TYPE_BANK1, }, - { "in_da_unknown", 4, 0x0e, STATS_TYPE_BANK1, }, - { "in_management", 4, 0x0f, STATS_TYPE_BANK1, }, - { "out_queue_0", 4, 0x10, STATS_TYPE_BANK1, }, - { "out_queue_1", 4, 0x11, STATS_TYPE_BANK1, }, - { "out_queue_2", 4, 0x12, STATS_TYPE_BANK1, }, - { "out_queue_3", 4, 0x13, STATS_TYPE_BANK1, }, - { "out_queue_4", 4, 0x14, STATS_TYPE_BANK1, }, - { "out_queue_5", 4, 0x15, STATS_TYPE_BANK1, }, - { "out_queue_6", 4, 0x16, STATS_TYPE_BANK1, }, - { "out_queue_7", 4, 0x17, STATS_TYPE_BANK1, }, - { "out_cut_through", 4, 0x18, STATS_TYPE_BANK1, }, - { "out_octets_a", 4, 0x1a, STATS_TYPE_BANK1, }, - { "out_octets_b", 4, 0x1b, STATS_TYPE_BANK1, }, - { "out_management", 4, 0x1f, STATS_TYPE_BANK1, }, + mv88e6xxx_reg_lock(chip); + err = chip->info->ops->stats_snapshot(chip, port); + mv88e6xxx_reg_unlock(chip); + + return err; +} + +#define MV88E6XXX_HW_STAT_MAPPER(_fn) \ + _fn(in_good_octets, 8, 0x00, STATS_TYPE_BANK0), \ + _fn(in_bad_octets, 4, 0x02, STATS_TYPE_BANK0), \ + _fn(in_unicast, 4, 0x04, STATS_TYPE_BANK0), \ + _fn(in_broadcasts, 4, 0x06, STATS_TYPE_BANK0), \ + _fn(in_multicasts, 4, 0x07, STATS_TYPE_BANK0), \ + _fn(in_pause, 4, 0x16, STATS_TYPE_BANK0), \ + _fn(in_undersize, 4, 0x18, STATS_TYPE_BANK0), \ + _fn(in_fragments, 4, 0x19, STATS_TYPE_BANK0), \ + _fn(in_oversize, 4, 0x1a, STATS_TYPE_BANK0), \ + _fn(in_jabber, 4, 0x1b, STATS_TYPE_BANK0), \ + _fn(in_rx_error, 4, 0x1c, STATS_TYPE_BANK0), \ + _fn(in_fcs_error, 4, 0x1d, STATS_TYPE_BANK0), \ + _fn(out_octets, 8, 0x0e, STATS_TYPE_BANK0), \ + _fn(out_unicast, 4, 0x10, STATS_TYPE_BANK0), \ + _fn(out_broadcasts, 4, 0x13, STATS_TYPE_BANK0), \ + _fn(out_multicasts, 4, 0x12, STATS_TYPE_BANK0), \ + _fn(out_pause, 4, 0x15, STATS_TYPE_BANK0), \ + _fn(excessive, 4, 0x11, STATS_TYPE_BANK0), \ + _fn(collisions, 4, 0x1e, STATS_TYPE_BANK0), \ + _fn(deferred, 4, 0x05, STATS_TYPE_BANK0), \ + _fn(single, 4, 0x14, STATS_TYPE_BANK0), \ + _fn(multiple, 4, 0x17, STATS_TYPE_BANK0), \ + _fn(out_fcs_error, 4, 0x03, STATS_TYPE_BANK0), \ + _fn(late, 4, 0x1f, STATS_TYPE_BANK0), \ + _fn(hist_64bytes, 4, 0x08, STATS_TYPE_BANK0), \ + _fn(hist_65_127bytes, 4, 0x09, STATS_TYPE_BANK0), \ + _fn(hist_128_255bytes, 4, 0x0a, STATS_TYPE_BANK0), \ + _fn(hist_256_511bytes, 4, 0x0b, STATS_TYPE_BANK0), \ + _fn(hist_512_1023bytes, 4, 0x0c, STATS_TYPE_BANK0), \ + _fn(hist_1024_max_bytes, 4, 0x0d, STATS_TYPE_BANK0), \ + _fn(sw_in_discards, 4, 0x10, STATS_TYPE_PORT), \ + _fn(sw_in_filtered, 2, 0x12, STATS_TYPE_PORT), \ + _fn(sw_out_filtered, 2, 0x13, STATS_TYPE_PORT), \ + _fn(in_discards, 4, 0x00, STATS_TYPE_BANK1), \ + _fn(in_filtered, 4, 0x01, STATS_TYPE_BANK1), \ + _fn(in_accepted, 4, 0x02, STATS_TYPE_BANK1), \ + _fn(in_bad_accepted, 4, 0x03, STATS_TYPE_BANK1), \ + _fn(in_good_avb_class_a, 4, 0x04, STATS_TYPE_BANK1), \ + _fn(in_good_avb_class_b, 4, 0x05, STATS_TYPE_BANK1), \ + _fn(in_bad_avb_class_a, 4, 0x06, STATS_TYPE_BANK1), \ + _fn(in_bad_avb_class_b, 4, 0x07, STATS_TYPE_BANK1), \ + _fn(tcam_counter_0, 4, 0x08, STATS_TYPE_BANK1), \ + _fn(tcam_counter_1, 4, 0x09, STATS_TYPE_BANK1), \ + _fn(tcam_counter_2, 4, 0x0a, STATS_TYPE_BANK1), \ + _fn(tcam_counter_3, 4, 0x0b, STATS_TYPE_BANK1), \ + _fn(in_da_unknown, 4, 0x0e, STATS_TYPE_BANK1), \ + _fn(in_management, 4, 0x0f, STATS_TYPE_BANK1), \ + _fn(out_queue_0, 4, 0x10, STATS_TYPE_BANK1), \ + _fn(out_queue_1, 4, 0x11, STATS_TYPE_BANK1), \ + _fn(out_queue_2, 4, 0x12, STATS_TYPE_BANK1), \ + _fn(out_queue_3, 4, 0x13, STATS_TYPE_BANK1), \ + _fn(out_queue_4, 4, 0x14, STATS_TYPE_BANK1), \ + _fn(out_queue_5, 4, 0x15, STATS_TYPE_BANK1), \ + _fn(out_queue_6, 4, 0x16, STATS_TYPE_BANK1), \ + _fn(out_queue_7, 4, 0x17, STATS_TYPE_BANK1), \ + _fn(out_cut_through, 4, 0x18, STATS_TYPE_BANK1), \ + _fn(out_octets_a, 4, 0x1a, STATS_TYPE_BANK1), \ + _fn(out_octets_b, 4, 0x1b, STATS_TYPE_BANK1), \ + _fn(out_management, 4, 0x1f, STATS_TYPE_BANK1), \ + /* */ + +#define MV88E6XXX_HW_STAT_ENTRY(_string, _size, _reg, _type) \ + { #_string, _size, _reg, _type } +static const struct mv88e6xxx_hw_stat mv88e6xxx_hw_stats[] = { + MV88E6XXX_HW_STAT_MAPPER(MV88E6XXX_HW_STAT_ENTRY) +}; + +#define MV88E6XXX_HW_STAT_ENUM(_string, _size, _reg, _type) \ + MV88E6XXX_HW_STAT_ID_ ## _string +enum mv88e6xxx_hw_stat_id { + MV88E6XXX_HW_STAT_MAPPER(MV88E6XXX_HW_STAT_ENUM) }; static uint64_t _mv88e6xxx_get_ethtool_stat(struct mv88e6xxx_chip *chip, - struct mv88e6xxx_hw_stat *s, + const struct mv88e6xxx_hw_stat *s, int port, u16 bank1_select, u16 histogram) { @@ -1055,7 +1073,7 @@ static uint64_t _mv88e6xxx_get_ethtool_stat(struct mv88e6xxx_chip *chip, static int mv88e6xxx_stats_get_strings(struct mv88e6xxx_chip *chip, uint8_t *data, int types) { - struct mv88e6xxx_hw_stat *stat; + const struct mv88e6xxx_hw_stat *stat; int i, j; for (i = 0, j = 0; i < ARRAY_SIZE(mv88e6xxx_hw_stats); i++) { @@ -1136,7 +1154,7 @@ static void mv88e6xxx_get_strings(struct dsa_switch *ds, int port, static int mv88e6xxx_stats_get_sset_count(struct mv88e6xxx_chip *chip, int types) { - struct mv88e6xxx_hw_stat *stat; + const struct mv88e6xxx_hw_stat *stat; int i, j; for (i = 0, j = 0; i < ARRAY_SIZE(mv88e6xxx_hw_stats); i++) { @@ -1195,59 +1213,82 @@ static int mv88e6xxx_get_sset_count(struct dsa_switch *ds, int port, int sset) return count; } -static int mv88e6xxx_stats_get_stats(struct mv88e6xxx_chip *chip, int port, - uint64_t *data, int types, - u16 bank1_select, u16 histogram) +static size_t mv88e6095_stats_get_stat(struct mv88e6xxx_chip *chip, int port, + const struct mv88e6xxx_hw_stat *stat, + uint64_t *data) { - struct mv88e6xxx_hw_stat *stat; - int i, j; + if (!(stat->type & (STATS_TYPE_BANK0 | STATS_TYPE_PORT))) + return 0; - for (i = 0, j = 0; i < ARRAY_SIZE(mv88e6xxx_hw_stats); i++) { - stat = &mv88e6xxx_hw_stats[i]; - if (stat->type & types) { - mv88e6xxx_reg_lock(chip); - data[j] = _mv88e6xxx_get_ethtool_stat(chip, stat, port, - bank1_select, - histogram); - mv88e6xxx_reg_unlock(chip); + *data = _mv88e6xxx_get_ethtool_stat(chip, stat, port, 0, + MV88E6XXX_G1_STATS_OP_HIST_RX); + return 1; +} - j++; - } - } - return j; +static size_t mv88e6250_stats_get_stat(struct mv88e6xxx_chip *chip, int port, + const struct mv88e6xxx_hw_stat *stat, + uint64_t *data) +{ + if (!(stat->type & STATS_TYPE_BANK0)) + return 0; + + *data = _mv88e6xxx_get_ethtool_stat(chip, stat, port, 0, + MV88E6XXX_G1_STATS_OP_HIST_RX); + return 1; } -static int mv88e6095_stats_get_stats(struct mv88e6xxx_chip *chip, int port, - uint64_t *data) +static size_t mv88e6320_stats_get_stat(struct mv88e6xxx_chip *chip, int port, + const struct mv88e6xxx_hw_stat *stat, + uint64_t *data) { - return mv88e6xxx_stats_get_stats(chip, port, data, - STATS_TYPE_BANK0 | STATS_TYPE_PORT, - 0, MV88E6XXX_G1_STATS_OP_HIST_RX_TX); + if (!(stat->type & (STATS_TYPE_BANK0 | STATS_TYPE_BANK1))) + return 0; + + *data = _mv88e6xxx_get_ethtool_stat(chip, stat, port, + MV88E6XXX_G1_STATS_OP_BANK_1_BIT_9, + MV88E6XXX_G1_STATS_OP_HIST_RX); + return 1; } -static int mv88e6250_stats_get_stats(struct mv88e6xxx_chip *chip, int port, - uint64_t *data) +static size_t mv88e6390_stats_get_stat(struct mv88e6xxx_chip *chip, int port, + const struct mv88e6xxx_hw_stat *stat, + uint64_t *data) { - return mv88e6xxx_stats_get_stats(chip, port, data, STATS_TYPE_BANK0, - 0, MV88E6XXX_G1_STATS_OP_HIST_RX_TX); + if (!(stat->type & (STATS_TYPE_BANK0 | STATS_TYPE_BANK1))) + return 0; + + *data = _mv88e6xxx_get_ethtool_stat(chip, stat, port, + MV88E6XXX_G1_STATS_OP_BANK_1_BIT_10, + 0); + return 1; } -static int mv88e6320_stats_get_stats(struct mv88e6xxx_chip *chip, int port, - uint64_t *data) +static size_t mv88e6xxx_stats_get_stat(struct mv88e6xxx_chip *chip, int port, + const struct mv88e6xxx_hw_stat *stat, + uint64_t *data) { - return mv88e6xxx_stats_get_stats(chip, port, data, - STATS_TYPE_BANK0 | STATS_TYPE_BANK1, - MV88E6XXX_G1_STATS_OP_BANK_1_BIT_9, - MV88E6XXX_G1_STATS_OP_HIST_RX_TX); + int ret = 0; + + if (chip->info->ops->stats_get_stat) { + mv88e6xxx_reg_lock(chip); + ret = chip->info->ops->stats_get_stat(chip, port, stat, data); + mv88e6xxx_reg_unlock(chip); + } + + return ret; } -static int mv88e6390_stats_get_stats(struct mv88e6xxx_chip *chip, int port, - uint64_t *data) +static size_t mv88e6xxx_stats_get_stats(struct mv88e6xxx_chip *chip, int port, + uint64_t *data) { - return mv88e6xxx_stats_get_stats(chip, port, data, - STATS_TYPE_BANK0 | STATS_TYPE_BANK1, - MV88E6XXX_G1_STATS_OP_BANK_1_BIT_10, - 0); + const struct mv88e6xxx_hw_stat *stat; + size_t i, j; + + for (i = 0, j = 0; i < ARRAY_SIZE(mv88e6xxx_hw_stats); i++) { + stat = &mv88e6xxx_hw_stats[i]; + j += mv88e6xxx_stats_get_stat(chip, port, stat, &data[j]); + } + return j; } static void mv88e6xxx_atu_vtu_get_stats(struct mv88e6xxx_chip *chip, int port, @@ -1263,10 +1304,9 @@ static void mv88e6xxx_atu_vtu_get_stats(struct mv88e6xxx_chip *chip, int port, static void mv88e6xxx_get_stats(struct mv88e6xxx_chip *chip, int port, uint64_t *data) { - int count = 0; + size_t count; - if (chip->info->ops->stats_get_stats) - count = chip->info->ops->stats_get_stats(chip, port, data); + count = mv88e6xxx_stats_get_stats(chip, port, data); mv88e6xxx_reg_lock(chip); if (chip->info->ops->serdes_get_stats) { @@ -1284,16 +1324,90 @@ static void mv88e6xxx_get_ethtool_stats(struct dsa_switch *ds, int port, struct mv88e6xxx_chip *chip = ds->priv; int ret; - mv88e6xxx_reg_lock(chip); + ret = mv88e6xxx_stats_snapshot(chip, port); + if (ret < 0) + return; + + mv88e6xxx_get_stats(chip, port, data); +} + +static void mv88e6xxx_get_eth_mac_stats(struct dsa_switch *ds, int port, + struct ethtool_eth_mac_stats *mac_stats) +{ + struct mv88e6xxx_chip *chip = ds->priv; + int ret; ret = mv88e6xxx_stats_snapshot(chip, port); - mv88e6xxx_reg_unlock(chip); + if (ret < 0) + return; +#define MV88E6XXX_ETH_MAC_STAT_MAP(_id, _member) \ + mv88e6xxx_stats_get_stat(chip, port, \ + &mv88e6xxx_hw_stats[MV88E6XXX_HW_STAT_ID_ ## _id], \ + &mac_stats->stats._member) + + MV88E6XXX_ETH_MAC_STAT_MAP(out_unicast, FramesTransmittedOK); + MV88E6XXX_ETH_MAC_STAT_MAP(single, SingleCollisionFrames); + MV88E6XXX_ETH_MAC_STAT_MAP(multiple, MultipleCollisionFrames); + MV88E6XXX_ETH_MAC_STAT_MAP(in_unicast, FramesReceivedOK); + MV88E6XXX_ETH_MAC_STAT_MAP(in_fcs_error, FrameCheckSequenceErrors); + MV88E6XXX_ETH_MAC_STAT_MAP(out_octets, OctetsTransmittedOK); + MV88E6XXX_ETH_MAC_STAT_MAP(deferred, FramesWithDeferredXmissions); + MV88E6XXX_ETH_MAC_STAT_MAP(late, LateCollisions); + MV88E6XXX_ETH_MAC_STAT_MAP(in_good_octets, OctetsReceivedOK); + MV88E6XXX_ETH_MAC_STAT_MAP(out_multicasts, MulticastFramesXmittedOK); + MV88E6XXX_ETH_MAC_STAT_MAP(out_broadcasts, BroadcastFramesXmittedOK); + MV88E6XXX_ETH_MAC_STAT_MAP(excessive, FramesWithExcessiveDeferral); + MV88E6XXX_ETH_MAC_STAT_MAP(in_multicasts, MulticastFramesReceivedOK); + MV88E6XXX_ETH_MAC_STAT_MAP(in_broadcasts, BroadcastFramesReceivedOK); + +#undef MV88E6XXX_ETH_MAC_STAT_MAP + + mac_stats->stats.FramesTransmittedOK += mac_stats->stats.MulticastFramesXmittedOK; + mac_stats->stats.FramesTransmittedOK += mac_stats->stats.BroadcastFramesXmittedOK; + mac_stats->stats.FramesReceivedOK += mac_stats->stats.MulticastFramesReceivedOK; + mac_stats->stats.FramesReceivedOK += mac_stats->stats.BroadcastFramesReceivedOK; +} + +static void mv88e6xxx_get_rmon_stats(struct dsa_switch *ds, int port, + struct ethtool_rmon_stats *rmon_stats, + const struct ethtool_rmon_hist_range **ranges) +{ + static const struct ethtool_rmon_hist_range rmon_ranges[] = { + { 64, 64 }, + { 65, 127 }, + { 128, 255 }, + { 256, 511 }, + { 512, 1023 }, + { 1024, 65535 }, + {} + }; + struct mv88e6xxx_chip *chip = ds->priv; + int ret; + + ret = mv88e6xxx_stats_snapshot(chip, port); if (ret < 0) return; - mv88e6xxx_get_stats(chip, port, data); +#define MV88E6XXX_RMON_STAT_MAP(_id, _member) \ + mv88e6xxx_stats_get_stat(chip, port, \ + &mv88e6xxx_hw_stats[MV88E6XXX_HW_STAT_ID_ ## _id], \ + &rmon_stats->stats._member) + + MV88E6XXX_RMON_STAT_MAP(in_undersize, undersize_pkts); + MV88E6XXX_RMON_STAT_MAP(in_oversize, oversize_pkts); + MV88E6XXX_RMON_STAT_MAP(in_fragments, fragments); + MV88E6XXX_RMON_STAT_MAP(in_jabber, jabbers); + MV88E6XXX_RMON_STAT_MAP(hist_64bytes, hist[0]); + MV88E6XXX_RMON_STAT_MAP(hist_65_127bytes, hist[1]); + MV88E6XXX_RMON_STAT_MAP(hist_128_255bytes, hist[2]); + MV88E6XXX_RMON_STAT_MAP(hist_256_511bytes, hist[3]); + MV88E6XXX_RMON_STAT_MAP(hist_512_1023bytes, hist[4]); + MV88E6XXX_RMON_STAT_MAP(hist_1024_max_bytes, hist[5]); + +#undef MV88E6XXX_RMON_STAT_MAP + *ranges = rmon_ranges; } static int mv88e6xxx_get_regs_len(struct dsa_switch *ds, int port) @@ -3987,7 +4101,7 @@ static const struct mv88e6xxx_ops mv88e6085_ops = { .stats_set_histogram = mv88e6095_g1_stats_set_histogram, .stats_get_sset_count = mv88e6095_stats_get_sset_count, .stats_get_strings = mv88e6095_stats_get_strings, - .stats_get_stats = mv88e6095_stats_get_stats, + .stats_get_stat = mv88e6095_stats_get_stat, .set_cpu_port = mv88e6095_g1_set_cpu_port, .set_egress_port = mv88e6095_g1_set_egress_port, .watchdog_ops = &mv88e6097_watchdog_ops, @@ -4025,7 +4139,7 @@ static const struct mv88e6xxx_ops mv88e6095_ops = { .stats_set_histogram = mv88e6095_g1_stats_set_histogram, .stats_get_sset_count = mv88e6095_stats_get_sset_count, .stats_get_strings = mv88e6095_stats_get_strings, - .stats_get_stats = mv88e6095_stats_get_stats, + .stats_get_stat = mv88e6095_stats_get_stat, .mgmt_rsvd2cpu = mv88e6185_g2_mgmt_rsvd2cpu, .ppu_enable = mv88e6185_g1_ppu_enable, .ppu_disable = mv88e6185_g1_ppu_disable, @@ -4066,7 +4180,7 @@ static const struct mv88e6xxx_ops mv88e6097_ops = { .stats_set_histogram = mv88e6095_g1_stats_set_histogram, .stats_get_sset_count = mv88e6095_stats_get_sset_count, .stats_get_strings = mv88e6095_stats_get_strings, - .stats_get_stats = mv88e6095_stats_get_stats, + .stats_get_stat = mv88e6095_stats_get_stat, .set_cpu_port = mv88e6095_g1_set_cpu_port, .set_egress_port = mv88e6095_g1_set_egress_port, .watchdog_ops = &mv88e6097_watchdog_ops, @@ -4108,7 +4222,7 @@ static const struct mv88e6xxx_ops mv88e6123_ops = { .stats_set_histogram = mv88e6095_g1_stats_set_histogram, .stats_get_sset_count = mv88e6095_stats_get_sset_count, .stats_get_strings = mv88e6095_stats_get_strings, - .stats_get_stats = mv88e6095_stats_get_stats, + .stats_get_stat = mv88e6095_stats_get_stat, .set_cpu_port = mv88e6095_g1_set_cpu_port, .set_egress_port = mv88e6095_g1_set_egress_port, .watchdog_ops = &mv88e6097_watchdog_ops, @@ -4151,7 +4265,7 @@ static const struct mv88e6xxx_ops mv88e6131_ops = { .stats_set_histogram = mv88e6095_g1_stats_set_histogram, .stats_get_sset_count = mv88e6095_stats_get_sset_count, .stats_get_strings = mv88e6095_stats_get_strings, - .stats_get_stats = mv88e6095_stats_get_stats, + .stats_get_stat = mv88e6095_stats_get_stat, .set_cpu_port = mv88e6095_g1_set_cpu_port, .set_egress_port = mv88e6095_g1_set_egress_port, .watchdog_ops = &mv88e6097_watchdog_ops, @@ -4200,7 +4314,7 @@ static const struct mv88e6xxx_ops mv88e6141_ops = { .stats_set_histogram = mv88e6390_g1_stats_set_histogram, .stats_get_sset_count = mv88e6320_stats_get_sset_count, .stats_get_strings = mv88e6320_stats_get_strings, - .stats_get_stats = mv88e6390_stats_get_stats, + .stats_get_stat = mv88e6390_stats_get_stat, .set_cpu_port = mv88e6390_g1_set_cpu_port, .set_egress_port = mv88e6390_g1_set_egress_port, .watchdog_ops = &mv88e6390_watchdog_ops, @@ -4255,7 +4369,7 @@ static const struct mv88e6xxx_ops mv88e6161_ops = { .stats_set_histogram = mv88e6095_g1_stats_set_histogram, .stats_get_sset_count = mv88e6095_stats_get_sset_count, .stats_get_strings = mv88e6095_stats_get_strings, - .stats_get_stats = mv88e6095_stats_get_stats, + .stats_get_stat = mv88e6095_stats_get_stat, .set_cpu_port = mv88e6095_g1_set_cpu_port, .set_egress_port = mv88e6095_g1_set_egress_port, .watchdog_ops = &mv88e6097_watchdog_ops, @@ -4293,7 +4407,7 @@ static const struct mv88e6xxx_ops mv88e6165_ops = { .stats_set_histogram = mv88e6095_g1_stats_set_histogram, .stats_get_sset_count = mv88e6095_stats_get_sset_count, .stats_get_strings = mv88e6095_stats_get_strings, - .stats_get_stats = mv88e6095_stats_get_stats, + .stats_get_stat = mv88e6095_stats_get_stat, .set_cpu_port = mv88e6095_g1_set_cpu_port, .set_egress_port = mv88e6095_g1_set_egress_port, .watchdog_ops = &mv88e6097_watchdog_ops, @@ -4341,7 +4455,7 @@ static const struct mv88e6xxx_ops mv88e6171_ops = { .stats_set_histogram = mv88e6095_g1_stats_set_histogram, .stats_get_sset_count = mv88e6095_stats_get_sset_count, .stats_get_strings = mv88e6095_stats_get_strings, - .stats_get_stats = mv88e6095_stats_get_stats, + .stats_get_stat = mv88e6095_stats_get_stat, .set_cpu_port = mv88e6095_g1_set_cpu_port, .set_egress_port = mv88e6095_g1_set_egress_port, .watchdog_ops = &mv88e6097_watchdog_ops, @@ -4390,7 +4504,7 @@ static const struct mv88e6xxx_ops mv88e6172_ops = { .stats_set_histogram = mv88e6095_g1_stats_set_histogram, .stats_get_sset_count = mv88e6095_stats_get_sset_count, .stats_get_strings = mv88e6095_stats_get_strings, - .stats_get_stats = mv88e6095_stats_get_stats, + .stats_get_stat = mv88e6095_stats_get_stat, .set_cpu_port = mv88e6095_g1_set_cpu_port, .set_egress_port = mv88e6095_g1_set_egress_port, .watchdog_ops = &mv88e6097_watchdog_ops, @@ -4441,7 +4555,7 @@ static const struct mv88e6xxx_ops mv88e6175_ops = { .stats_set_histogram = mv88e6095_g1_stats_set_histogram, .stats_get_sset_count = mv88e6095_stats_get_sset_count, .stats_get_strings = mv88e6095_stats_get_strings, - .stats_get_stats = mv88e6095_stats_get_stats, + .stats_get_stat = mv88e6095_stats_get_stat, .set_cpu_port = mv88e6095_g1_set_cpu_port, .set_egress_port = mv88e6095_g1_set_egress_port, .watchdog_ops = &mv88e6097_watchdog_ops, @@ -4490,7 +4604,7 @@ static const struct mv88e6xxx_ops mv88e6176_ops = { .stats_set_histogram = mv88e6095_g1_stats_set_histogram, .stats_get_sset_count = mv88e6095_stats_get_sset_count, .stats_get_strings = mv88e6095_stats_get_strings, - .stats_get_stats = mv88e6095_stats_get_stats, + .stats_get_stat = mv88e6095_stats_get_stat, .set_cpu_port = mv88e6095_g1_set_cpu_port, .set_egress_port = mv88e6095_g1_set_egress_port, .watchdog_ops = &mv88e6097_watchdog_ops, @@ -4535,7 +4649,7 @@ static const struct mv88e6xxx_ops mv88e6185_ops = { .stats_set_histogram = mv88e6095_g1_stats_set_histogram, .stats_get_sset_count = mv88e6095_stats_get_sset_count, .stats_get_strings = mv88e6095_stats_get_strings, - .stats_get_stats = mv88e6095_stats_get_stats, + .stats_get_stat = mv88e6095_stats_get_stat, .set_cpu_port = mv88e6095_g1_set_cpu_port, .set_egress_port = mv88e6095_g1_set_egress_port, .watchdog_ops = &mv88e6097_watchdog_ops, @@ -4584,7 +4698,7 @@ static const struct mv88e6xxx_ops mv88e6190_ops = { .stats_set_histogram = mv88e6390_g1_stats_set_histogram, .stats_get_sset_count = mv88e6320_stats_get_sset_count, .stats_get_strings = mv88e6320_stats_get_strings, - .stats_get_stats = mv88e6390_stats_get_stats, + .stats_get_stat = mv88e6390_stats_get_stat, .set_cpu_port = mv88e6390_g1_set_cpu_port, .set_egress_port = mv88e6390_g1_set_egress_port, .watchdog_ops = &mv88e6390_watchdog_ops, @@ -4642,7 +4756,7 @@ static const struct mv88e6xxx_ops mv88e6190x_ops = { .stats_set_histogram = mv88e6390_g1_stats_set_histogram, .stats_get_sset_count = mv88e6320_stats_get_sset_count, .stats_get_strings = mv88e6320_stats_get_strings, - .stats_get_stats = mv88e6390_stats_get_stats, + .stats_get_stat = mv88e6390_stats_get_stat, .set_cpu_port = mv88e6390_g1_set_cpu_port, .set_egress_port = mv88e6390_g1_set_egress_port, .watchdog_ops = &mv88e6390_watchdog_ops, @@ -4698,7 +4812,7 @@ static const struct mv88e6xxx_ops mv88e6191_ops = { .stats_set_histogram = mv88e6390_g1_stats_set_histogram, .stats_get_sset_count = mv88e6320_stats_get_sset_count, .stats_get_strings = mv88e6320_stats_get_strings, - .stats_get_stats = mv88e6390_stats_get_stats, + .stats_get_stat = mv88e6390_stats_get_stat, .set_cpu_port = mv88e6390_g1_set_cpu_port, .set_egress_port = mv88e6390_g1_set_egress_port, .watchdog_ops = &mv88e6390_watchdog_ops, @@ -4757,7 +4871,7 @@ static const struct mv88e6xxx_ops mv88e6240_ops = { .stats_set_histogram = mv88e6095_g1_stats_set_histogram, .stats_get_sset_count = mv88e6095_stats_get_sset_count, .stats_get_strings = mv88e6095_stats_get_strings, - .stats_get_stats = mv88e6095_stats_get_stats, + .stats_get_stat = mv88e6095_stats_get_stat, .set_cpu_port = mv88e6095_g1_set_cpu_port, .set_egress_port = mv88e6095_g1_set_egress_port, .watchdog_ops = &mv88e6097_watchdog_ops, @@ -4810,7 +4924,7 @@ static const struct mv88e6xxx_ops mv88e6250_ops = { .stats_set_histogram = mv88e6095_g1_stats_set_histogram, .stats_get_sset_count = mv88e6250_stats_get_sset_count, .stats_get_strings = mv88e6250_stats_get_strings, - .stats_get_stats = mv88e6250_stats_get_stats, + .stats_get_stat = mv88e6250_stats_get_stat, .set_cpu_port = mv88e6095_g1_set_cpu_port, .set_egress_port = mv88e6095_g1_set_egress_port, .watchdog_ops = &mv88e6250_watchdog_ops, @@ -4857,7 +4971,7 @@ static const struct mv88e6xxx_ops mv88e6290_ops = { .stats_set_histogram = mv88e6390_g1_stats_set_histogram, .stats_get_sset_count = mv88e6320_stats_get_sset_count, .stats_get_strings = mv88e6320_stats_get_strings, - .stats_get_stats = mv88e6390_stats_get_stats, + .stats_get_stat = mv88e6390_stats_get_stat, .set_cpu_port = mv88e6390_g1_set_cpu_port, .set_egress_port = mv88e6390_g1_set_egress_port, .watchdog_ops = &mv88e6390_watchdog_ops, @@ -4916,7 +5030,7 @@ static const struct mv88e6xxx_ops mv88e6320_ops = { .stats_set_histogram = mv88e6095_g1_stats_set_histogram, .stats_get_sset_count = mv88e6320_stats_get_sset_count, .stats_get_strings = mv88e6320_stats_get_strings, - .stats_get_stats = mv88e6320_stats_get_stats, + .stats_get_stat = mv88e6320_stats_get_stat, .set_cpu_port = mv88e6095_g1_set_cpu_port, .set_egress_port = mv88e6095_g1_set_egress_port, .watchdog_ops = &mv88e6390_watchdog_ops, @@ -4963,7 +5077,7 @@ static const struct mv88e6xxx_ops mv88e6321_ops = { .stats_set_histogram = mv88e6095_g1_stats_set_histogram, .stats_get_sset_count = mv88e6320_stats_get_sset_count, .stats_get_strings = mv88e6320_stats_get_strings, - .stats_get_stats = mv88e6320_stats_get_stats, + .stats_get_stat = mv88e6320_stats_get_stat, .set_cpu_port = mv88e6095_g1_set_cpu_port, .set_egress_port = mv88e6095_g1_set_egress_port, .watchdog_ops = &mv88e6390_watchdog_ops, @@ -5012,7 +5126,7 @@ static const struct mv88e6xxx_ops mv88e6341_ops = { .stats_set_histogram = mv88e6390_g1_stats_set_histogram, .stats_get_sset_count = mv88e6320_stats_get_sset_count, .stats_get_strings = mv88e6320_stats_get_strings, - .stats_get_stats = mv88e6390_stats_get_stats, + .stats_get_stat = mv88e6390_stats_get_stat, .set_cpu_port = mv88e6390_g1_set_cpu_port, .set_egress_port = mv88e6390_g1_set_egress_port, .watchdog_ops = &mv88e6390_watchdog_ops, @@ -5070,7 +5184,7 @@ static const struct mv88e6xxx_ops mv88e6350_ops = { .stats_set_histogram = mv88e6095_g1_stats_set_histogram, .stats_get_sset_count = mv88e6095_stats_get_sset_count, .stats_get_strings = mv88e6095_stats_get_strings, - .stats_get_stats = mv88e6095_stats_get_stats, + .stats_get_stat = mv88e6095_stats_get_stat, .set_cpu_port = mv88e6095_g1_set_cpu_port, .set_egress_port = mv88e6095_g1_set_egress_port, .watchdog_ops = &mv88e6097_watchdog_ops, @@ -5116,7 +5230,7 @@ static const struct mv88e6xxx_ops mv88e6351_ops = { .stats_set_histogram = mv88e6095_g1_stats_set_histogram, .stats_get_sset_count = mv88e6095_stats_get_sset_count, .stats_get_strings = mv88e6095_stats_get_strings, - .stats_get_stats = mv88e6095_stats_get_stats, + .stats_get_stat = mv88e6095_stats_get_stat, .set_cpu_port = mv88e6095_g1_set_cpu_port, .set_egress_port = mv88e6095_g1_set_egress_port, .watchdog_ops = &mv88e6097_watchdog_ops, @@ -5167,7 +5281,7 @@ static const struct mv88e6xxx_ops mv88e6352_ops = { .stats_set_histogram = mv88e6095_g1_stats_set_histogram, .stats_get_sset_count = mv88e6095_stats_get_sset_count, .stats_get_strings = mv88e6095_stats_get_strings, - .stats_get_stats = mv88e6095_stats_get_stats, + .stats_get_stat = mv88e6095_stats_get_stat, .set_cpu_port = mv88e6095_g1_set_cpu_port, .set_egress_port = mv88e6095_g1_set_egress_port, .watchdog_ops = &mv88e6097_watchdog_ops, @@ -5229,7 +5343,7 @@ static const struct mv88e6xxx_ops mv88e6390_ops = { .stats_set_histogram = mv88e6390_g1_stats_set_histogram, .stats_get_sset_count = mv88e6320_stats_get_sset_count, .stats_get_strings = mv88e6320_stats_get_strings, - .stats_get_stats = mv88e6390_stats_get_stats, + .stats_get_stat = mv88e6390_stats_get_stat, .set_cpu_port = mv88e6390_g1_set_cpu_port, .set_egress_port = mv88e6390_g1_set_egress_port, .watchdog_ops = &mv88e6390_watchdog_ops, @@ -5291,7 +5405,7 @@ static const struct mv88e6xxx_ops mv88e6390x_ops = { .stats_set_histogram = mv88e6390_g1_stats_set_histogram, .stats_get_sset_count = mv88e6320_stats_get_sset_count, .stats_get_strings = mv88e6320_stats_get_strings, - .stats_get_stats = mv88e6390_stats_get_stats, + .stats_get_stat = mv88e6390_stats_get_stat, .set_cpu_port = mv88e6390_g1_set_cpu_port, .set_egress_port = mv88e6390_g1_set_egress_port, .watchdog_ops = &mv88e6390_watchdog_ops, @@ -5353,7 +5467,7 @@ static const struct mv88e6xxx_ops mv88e6393x_ops = { .stats_set_histogram = mv88e6390_g1_stats_set_histogram, .stats_get_sset_count = mv88e6320_stats_get_sset_count, .stats_get_strings = mv88e6320_stats_get_strings, - .stats_get_stats = mv88e6390_stats_get_stats, + .stats_get_stat = mv88e6390_stats_get_stat, /* .set_cpu_port is missing because this family does not support a global * CPU port, only per port CPU port which is set via * .port_set_upstream_port method. @@ -6817,6 +6931,8 @@ static const struct dsa_switch_ops mv88e6xxx_switch_ops = { .phylink_mac_link_up = mv88e6xxx_mac_link_up, .get_strings = mv88e6xxx_get_strings, .get_ethtool_stats = mv88e6xxx_get_ethtool_stats, + .get_eth_mac_stats = mv88e6xxx_get_eth_mac_stats, + .get_rmon_stats = mv88e6xxx_get_rmon_stats, .get_sset_count = mv88e6xxx_get_sset_count, .port_max_mtu = mv88e6xxx_get_max_mtu, .port_change_mtu = mv88e6xxx_change_mtu, diff --git a/drivers/net/dsa/mv88e6xxx/chip.h b/drivers/net/dsa/mv88e6xxx/chip.h index 44383a03ef2ff9..85eb293381a7e2 100644 --- a/drivers/net/dsa/mv88e6xxx/chip.h +++ b/drivers/net/dsa/mv88e6xxx/chip.h @@ -318,6 +318,17 @@ struct mv88e6xxx_mst { struct mv88e6xxx_stu_entry stu; }; +#define STATS_TYPE_PORT BIT(0) +#define STATS_TYPE_BANK0 BIT(1) +#define STATS_TYPE_BANK1 BIT(2) + +struct mv88e6xxx_hw_stat { + char string[ETH_GSTRING_LEN]; + size_t size; + int reg; + int type; +}; + struct mv88e6xxx_chip { const struct mv88e6xxx_info *info; @@ -574,8 +585,9 @@ struct mv88e6xxx_ops { /* Return the number of strings describing statistics */ int (*stats_get_sset_count)(struct mv88e6xxx_chip *chip); int (*stats_get_strings)(struct mv88e6xxx_chip *chip, uint8_t *data); - int (*stats_get_stats)(struct mv88e6xxx_chip *chip, int port, - uint64_t *data); + size_t (*stats_get_stat)(struct mv88e6xxx_chip *chip, int port, + const struct mv88e6xxx_hw_stat *stat, + uint64_t *data); int (*set_cpu_port)(struct mv88e6xxx_chip *chip, int port); int (*set_egress_port)(struct mv88e6xxx_chip *chip, enum mv88e6xxx_egress_direction direction, @@ -601,8 +613,8 @@ struct mv88e6xxx_ops { int (*serdes_get_sset_count)(struct mv88e6xxx_chip *chip, int port); int (*serdes_get_strings)(struct mv88e6xxx_chip *chip, int port, uint8_t *data); - int (*serdes_get_stats)(struct mv88e6xxx_chip *chip, int port, - uint64_t *data); + size_t (*serdes_get_stats)(struct mv88e6xxx_chip *chip, int port, + uint64_t *data); /* SERDES registers for ethtool */ int (*serdes_get_regs_len)(struct mv88e6xxx_chip *chip, int port); @@ -727,17 +739,6 @@ struct mv88e6xxx_pcs_ops { }; -#define STATS_TYPE_PORT BIT(0) -#define STATS_TYPE_BANK0 BIT(1) -#define STATS_TYPE_BANK1 BIT(2) - -struct mv88e6xxx_hw_stat { - char string[ETH_GSTRING_LEN]; - size_t size; - int reg; - int type; -}; - static inline bool mv88e6xxx_has_stu(struct mv88e6xxx_chip *chip) { return chip->info->max_sid > 0 && diff --git a/drivers/net/dsa/mv88e6xxx/global1.c b/drivers/net/dsa/mv88e6xxx/global1.c index 174c773b38c2bd..49444a72ff095b 100644 --- a/drivers/net/dsa/mv88e6xxx/global1.c +++ b/drivers/net/dsa/mv88e6xxx/global1.c @@ -462,8 +462,7 @@ int mv88e6390_g1_rmu_disable(struct mv88e6xxx_chip *chip) int mv88e6390_g1_stats_set_histogram(struct mv88e6xxx_chip *chip) { return mv88e6xxx_g1_ctl2_mask(chip, MV88E6390_G1_CTL2_HIST_MODE_MASK, - MV88E6390_G1_CTL2_HIST_MODE_RX | - MV88E6390_G1_CTL2_HIST_MODE_TX); + MV88E6390_G1_CTL2_HIST_MODE_RX); } int mv88e6xxx_g1_set_device_number(struct mv88e6xxx_chip *chip, int index) @@ -491,7 +490,7 @@ int mv88e6095_g1_stats_set_histogram(struct mv88e6xxx_chip *chip) if (err) return err; - val |= MV88E6XXX_G1_STATS_OP_HIST_RX_TX; + val |= MV88E6XXX_G1_STATS_OP_HIST_RX; err = mv88e6xxx_g1_write(chip, MV88E6XXX_G1_STATS_OP, val); @@ -506,7 +505,7 @@ int mv88e6xxx_g1_stats_snapshot(struct mv88e6xxx_chip *chip, int port) err = mv88e6xxx_g1_write(chip, MV88E6XXX_G1_STATS_OP, MV88E6XXX_G1_STATS_OP_BUSY | MV88E6XXX_G1_STATS_OP_CAPTURE_PORT | - MV88E6XXX_G1_STATS_OP_HIST_RX_TX | port); + MV88E6XXX_G1_STATS_OP_HIST_RX | port); if (err) return err; diff --git a/drivers/net/dsa/mv88e6xxx/serdes.c b/drivers/net/dsa/mv88e6xxx/serdes.c index 3b4b42651fa3d7..01ea53940786d0 100644 --- a/drivers/net/dsa/mv88e6xxx/serdes.c +++ b/drivers/net/dsa/mv88e6xxx/serdes.c @@ -177,8 +177,8 @@ static uint64_t mv88e6352_serdes_get_stat(struct mv88e6xxx_chip *chip, return val; } -int mv88e6352_serdes_get_stats(struct mv88e6xxx_chip *chip, int port, - uint64_t *data) +size_t mv88e6352_serdes_get_stats(struct mv88e6xxx_chip *chip, int port, + uint64_t *data) { struct mv88e6xxx_port *mv88e6xxx_port = &chip->ports[port]; struct mv88e6352_serdes_hw_stat *stat; @@ -187,7 +187,7 @@ int mv88e6352_serdes_get_stats(struct mv88e6xxx_chip *chip, int port, err = mv88e6352_g2_scratch_port_has_serdes(chip, port); if (err <= 0) - return err; + return 0; BUILD_BUG_ON(ARRAY_SIZE(mv88e6352_serdes_hw_stats) > ARRAY_SIZE(mv88e6xxx_port->serdes_stats)); @@ -429,8 +429,8 @@ static uint64_t mv88e6390_serdes_get_stat(struct mv88e6xxx_chip *chip, int lane, return reg[0] | ((u64)reg[1] << 16) | ((u64)reg[2] << 32); } -int mv88e6390_serdes_get_stats(struct mv88e6xxx_chip *chip, int port, - uint64_t *data) +size_t mv88e6390_serdes_get_stats(struct mv88e6xxx_chip *chip, int port, + uint64_t *data) { struct mv88e6390_serdes_hw_stat *stat; int lane; diff --git a/drivers/net/dsa/mv88e6xxx/serdes.h b/drivers/net/dsa/mv88e6xxx/serdes.h index aac95cab46e3de..ff5c3ab31e155e 100644 --- a/drivers/net/dsa/mv88e6xxx/serdes.h +++ b/drivers/net/dsa/mv88e6xxx/serdes.h @@ -127,13 +127,13 @@ unsigned int mv88e6390_serdes_irq_mapping(struct mv88e6xxx_chip *chip, int mv88e6352_serdes_get_sset_count(struct mv88e6xxx_chip *chip, int port); int mv88e6352_serdes_get_strings(struct mv88e6xxx_chip *chip, int port, uint8_t *data); -int mv88e6352_serdes_get_stats(struct mv88e6xxx_chip *chip, int port, - uint64_t *data); +size_t mv88e6352_serdes_get_stats(struct mv88e6xxx_chip *chip, int port, + uint64_t *data); int mv88e6390_serdes_get_sset_count(struct mv88e6xxx_chip *chip, int port); int mv88e6390_serdes_get_strings(struct mv88e6xxx_chip *chip, int port, uint8_t *data); -int mv88e6390_serdes_get_stats(struct mv88e6xxx_chip *chip, int port, - uint64_t *data); +size_t mv88e6390_serdes_get_stats(struct mv88e6xxx_chip *chip, int port, + uint64_t *data); int mv88e6352_serdes_get_regs_len(struct mv88e6xxx_chip *chip, int port); void mv88e6352_serdes_get_regs(struct mv88e6xxx_chip *chip, int port, void *_p); diff --git a/drivers/net/ethernet/intel/ice/Makefile b/drivers/net/ethernet/intel/ice/Makefile index b40b4179b9f4d1..cddd82d4ca0f3f 100644 --- a/drivers/net/ethernet/intel/ice/Makefile +++ b/drivers/net/ethernet/intel/ice/Makefile @@ -34,7 +34,9 @@ ice-y := ice_main.o \ ice_lag.o \ ice_ethtool.o \ ice_repr.o \ - ice_tc_lib.o + ice_tc_lib.o \ + ice_fwlog.o \ + ice_debugfs.o ice-$(CONFIG_PCI_IOV) += \ ice_sriov.o \ ice_virtchnl.o \ diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h index 2b3aa37df4f365..2defac6d91680a 100644 --- a/drivers/net/ethernet/intel/ice/ice.h +++ b/drivers/net/ethernet/intel/ice/ice.h @@ -571,6 +571,10 @@ struct ice_pf { struct ice_vsi_stats **vsi_stats; struct ice_sw *first_sw; /* first switch created by firmware */ u16 eswitch_mode; /* current mode of eswitch */ + struct dentry *ice_debugfs_pf; + struct dentry *ice_debugfs_pf_fwlog; + /* keep track of all the dentrys for FW log modules */ + struct dentry **ice_debugfs_pf_fwlog_modules; struct ice_vfs vfs; DECLARE_BITMAP(features, ICE_F_MAX); DECLARE_BITMAP(state, ICE_STATE_NBITS); @@ -890,6 +894,11 @@ static inline bool ice_is_adq_active(struct ice_pf *pf) return false; } +void ice_debugfs_fwlog_init(struct ice_pf *pf); +void ice_debugfs_init(void); +void ice_debugfs_exit(void); +void ice_pf_fwlog_update_module(struct ice_pf *pf, int log_level, int module); + bool netif_is_ice(const struct net_device *dev); int ice_vsi_setup_tx_rings(struct ice_vsi *vsi); int ice_vsi_setup_rx_rings(struct ice_vsi *vsi); @@ -999,4 +1008,6 @@ static inline void ice_clear_rdma_cap(struct ice_pf *pf) set_bit(ICE_FLAG_UNPLUG_AUX_DEV, pf->flags); clear_bit(ICE_FLAG_RDMA_ENA, pf->flags); } + +extern const struct xdp_metadata_ops ice_xdp_md_ops; #endif /* _ICE_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h b/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h index adf7a5c78f8594..12c510bb1d9b5f 100644 --- a/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h +++ b/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h @@ -2094,78 +2094,6 @@ struct ice_aqc_add_rdma_qset_data { struct ice_aqc_add_tx_rdma_qset_entry rdma_qsets[]; }; -/* Configure Firmware Logging Command (indirect 0xFF09) - * Logging Information Read Response (indirect 0xFF10) - * Note: The 0xFF10 command has no input parameters. - */ -struct ice_aqc_fw_logging { - u8 log_ctrl; -#define ICE_AQC_FW_LOG_AQ_EN BIT(0) -#define ICE_AQC_FW_LOG_UART_EN BIT(1) - u8 rsvd0; - u8 log_ctrl_valid; /* Not used by 0xFF10 Response */ -#define ICE_AQC_FW_LOG_AQ_VALID BIT(0) -#define ICE_AQC_FW_LOG_UART_VALID BIT(1) - u8 rsvd1[5]; - __le32 addr_high; - __le32 addr_low; -}; - -enum ice_aqc_fw_logging_mod { - ICE_AQC_FW_LOG_ID_GENERAL = 0, - ICE_AQC_FW_LOG_ID_CTRL, - ICE_AQC_FW_LOG_ID_LINK, - ICE_AQC_FW_LOG_ID_LINK_TOPO, - ICE_AQC_FW_LOG_ID_DNL, - ICE_AQC_FW_LOG_ID_I2C, - ICE_AQC_FW_LOG_ID_SDP, - ICE_AQC_FW_LOG_ID_MDIO, - ICE_AQC_FW_LOG_ID_ADMINQ, - ICE_AQC_FW_LOG_ID_HDMA, - ICE_AQC_FW_LOG_ID_LLDP, - ICE_AQC_FW_LOG_ID_DCBX, - ICE_AQC_FW_LOG_ID_DCB, - ICE_AQC_FW_LOG_ID_NETPROXY, - ICE_AQC_FW_LOG_ID_NVM, - ICE_AQC_FW_LOG_ID_AUTH, - ICE_AQC_FW_LOG_ID_VPD, - ICE_AQC_FW_LOG_ID_IOSF, - ICE_AQC_FW_LOG_ID_PARSER, - ICE_AQC_FW_LOG_ID_SW, - ICE_AQC_FW_LOG_ID_SCHEDULER, - ICE_AQC_FW_LOG_ID_TXQ, - ICE_AQC_FW_LOG_ID_RSVD, - ICE_AQC_FW_LOG_ID_POST, - ICE_AQC_FW_LOG_ID_WATCHDOG, - ICE_AQC_FW_LOG_ID_TASK_DISPATCH, - ICE_AQC_FW_LOG_ID_MNG, - ICE_AQC_FW_LOG_ID_MAX, -}; - -/* Defines for both above FW logging command/response buffers */ -#define ICE_AQC_FW_LOG_ID_S 0 -#define ICE_AQC_FW_LOG_ID_M (0xFFF << ICE_AQC_FW_LOG_ID_S) - -#define ICE_AQC_FW_LOG_CONF_SUCCESS 0 /* Used by response */ -#define ICE_AQC_FW_LOG_CONF_BAD_INDX BIT(12) /* Used by response */ - -#define ICE_AQC_FW_LOG_EN_S 12 -#define ICE_AQC_FW_LOG_EN_M (0xF << ICE_AQC_FW_LOG_EN_S) -#define ICE_AQC_FW_LOG_INFO_EN BIT(12) /* Used by command */ -#define ICE_AQC_FW_LOG_INIT_EN BIT(13) /* Used by command */ -#define ICE_AQC_FW_LOG_FLOW_EN BIT(14) /* Used by command */ -#define ICE_AQC_FW_LOG_ERR_EN BIT(15) /* Used by command */ - -/* Get/Clear FW Log (indirect 0xFF11) */ -struct ice_aqc_get_clear_fw_log { - u8 flags; -#define ICE_AQC_FW_LOG_CLEAR BIT(0) -#define ICE_AQC_FW_LOG_MORE_DATA_AVAIL BIT(1) - u8 rsvd1[7]; - __le32 addr_high; - __le32 addr_low; -}; - /* Download Package (indirect 0x0C40) */ /* Also used for Update Package (indirect 0x0C41 and 0x0C42) */ struct ice_aqc_download_pkg { @@ -2428,6 +2356,84 @@ struct ice_aqc_event_lan_overflow { u8 reserved[8]; }; +enum ice_aqc_fw_logging_mod { + ICE_AQC_FW_LOG_ID_GENERAL = 0, + ICE_AQC_FW_LOG_ID_CTRL, + ICE_AQC_FW_LOG_ID_LINK, + ICE_AQC_FW_LOG_ID_LINK_TOPO, + ICE_AQC_FW_LOG_ID_DNL, + ICE_AQC_FW_LOG_ID_I2C, + ICE_AQC_FW_LOG_ID_SDP, + ICE_AQC_FW_LOG_ID_MDIO, + ICE_AQC_FW_LOG_ID_ADMINQ, + ICE_AQC_FW_LOG_ID_HDMA, + ICE_AQC_FW_LOG_ID_LLDP, + ICE_AQC_FW_LOG_ID_DCBX, + ICE_AQC_FW_LOG_ID_DCB, + ICE_AQC_FW_LOG_ID_XLR, + ICE_AQC_FW_LOG_ID_NVM, + ICE_AQC_FW_LOG_ID_AUTH, + ICE_AQC_FW_LOG_ID_VPD, + ICE_AQC_FW_LOG_ID_IOSF, + ICE_AQC_FW_LOG_ID_PARSER, + ICE_AQC_FW_LOG_ID_SW, + ICE_AQC_FW_LOG_ID_SCHEDULER, + ICE_AQC_FW_LOG_ID_TXQ, + ICE_AQC_FW_LOG_ID_RSVD, + ICE_AQC_FW_LOG_ID_POST, + ICE_AQC_FW_LOG_ID_WATCHDOG, + ICE_AQC_FW_LOG_ID_TASK_DISPATCH, + ICE_AQC_FW_LOG_ID_MNG, + ICE_AQC_FW_LOG_ID_SYNCE, + ICE_AQC_FW_LOG_ID_HEALTH, + ICE_AQC_FW_LOG_ID_TSDRV, + ICE_AQC_FW_LOG_ID_PFREG, + ICE_AQC_FW_LOG_ID_MDLVER, + ICE_AQC_FW_LOG_ID_MAX, +}; + +/* Set FW Logging configuration (indirect 0xFF30) + * Register for FW Logging (indirect 0xFF31) + * Query FW Logging (indirect 0xFF32) + * FW Log Event (indirect 0xFF33) + */ +struct ice_aqc_fw_log { + u8 cmd_flags; +#define ICE_AQC_FW_LOG_CONF_UART_EN BIT(0) +#define ICE_AQC_FW_LOG_CONF_AQ_EN BIT(1) +#define ICE_AQC_FW_LOG_QUERY_REGISTERED BIT(2) +#define ICE_AQC_FW_LOG_CONF_SET_VALID BIT(3) +#define ICE_AQC_FW_LOG_AQ_REGISTER BIT(0) +#define ICE_AQC_FW_LOG_AQ_QUERY BIT(2) + + u8 rsp_flag; + __le16 fw_rt_msb; + union { + struct { + __le32 fw_rt_lsb; + } sync; + struct { + __le16 log_resolution; +#define ICE_AQC_FW_LOG_MIN_RESOLUTION (1) +#define ICE_AQC_FW_LOG_MAX_RESOLUTION (128) + + __le16 mdl_cnt; + } cfg; + } ops; + __le32 addr_high; + __le32 addr_low; +}; + +/* Response Buffer for: + * Set Firmware Logging Configuration (0xFF30) + * Query FW Logging (0xFF32) + */ +struct ice_aqc_fw_log_cfg_resp { + __le16 module_identifier; + u8 log_level; + u8 rsvd0; +}; + /** * struct ice_aq_desc - Admin Queue (AQ) descriptor * @flags: ICE_AQ_FLAG_* flags @@ -2507,8 +2513,6 @@ struct ice_aq_desc { struct ice_aqc_add_rdma_qset add_rdma_qset; struct ice_aqc_add_get_update_free_vsi vsi_cmd; struct ice_aqc_add_update_free_vsi_resp add_update_free_vsi_res; - struct ice_aqc_fw_logging fw_logging; - struct ice_aqc_get_clear_fw_log get_clear_fw_log; struct ice_aqc_download_pkg download_pkg; struct ice_aqc_set_cgu_input_config set_cgu_input_config; struct ice_aqc_get_cgu_input_config get_cgu_input_config; @@ -2520,6 +2524,7 @@ struct ice_aq_desc { struct ice_aqc_get_cgu_ref_prio get_cgu_ref_prio; struct ice_aqc_get_cgu_info get_cgu_info; struct ice_aqc_driver_shared_params drv_shared_params; + struct ice_aqc_fw_log fw_log; struct ice_aqc_set_mac_lb set_mac_lb; struct ice_aqc_alloc_free_res_cmd sw_res_ctrl; struct ice_aqc_set_mac_cfg set_mac_cfg; @@ -2718,9 +2723,11 @@ enum ice_adminq_opc { /* Standalone Commands/Events */ ice_aqc_opc_event_lan_overflow = 0x1001, - /* debug commands */ - ice_aqc_opc_fw_logging = 0xFF09, - ice_aqc_opc_fw_logging_info = 0xFF10, + /* FW Logging Commands */ + ice_aqc_opc_fw_logs_config = 0xFF30, + ice_aqc_opc_fw_logs_register = 0xFF31, + ice_aqc_opc_fw_logs_query = 0xFF32, + ice_aqc_opc_fw_logs_event = 0xFF33, }; #endif /* _ICE_ADMINQ_CMD_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_base.c b/drivers/net/ethernet/intel/ice/ice_base.c index edad5f9ab16ce0..b25b7f41596523 100644 --- a/drivers/net/ethernet/intel/ice/ice_base.c +++ b/drivers/net/ethernet/intel/ice/ice_base.c @@ -527,6 +527,19 @@ static int ice_setup_rx_ctx(struct ice_rx_ring *ring) return 0; } +static void ice_xsk_pool_fill_cb(struct ice_rx_ring *ring) +{ + void *ctx_ptr = &ring->pkt_ctx; + struct xsk_cb_desc desc = {}; + + XSK_CHECK_PRIV_TYPE(struct ice_xdp_buff); + desc.src = &ctx_ptr; + desc.off = offsetof(struct ice_xdp_buff, pkt_ctx) - + sizeof(struct xdp_buff); + desc.bytes = sizeof(ctx_ptr); + xsk_pool_fill_cb(ring->xsk_pool, &desc); +} + /** * ice_vsi_cfg_rxq - Configure an Rx queue * @ring: the ring being configured @@ -561,6 +574,7 @@ int ice_vsi_cfg_rxq(struct ice_rx_ring *ring) if (err) return err; xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq); + ice_xsk_pool_fill_cb(ring); dev_info(dev, "Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n", ring->q_index); @@ -583,6 +597,7 @@ int ice_vsi_cfg_rxq(struct ice_rx_ring *ring) xdp_init_buff(&ring->xdp, ice_rx_pg_size(ring) / 2, &ring->xdp_rxq); ring->xdp.data = NULL; + ring->xdp_ext.pkt_ctx = &ring->pkt_ctx; err = ice_setup_rx_ctx(ring); if (err) { dev_err(dev, "ice_setup_rx_ctx failed for RxQ %d, err %d\n", diff --git a/drivers/net/ethernet/intel/ice/ice_common.c b/drivers/net/ethernet/intel/ice/ice_common.c index 8d97434e1413e1..08a1f699a34fc0 100644 --- a/drivers/net/ethernet/intel/ice/ice_common.c +++ b/drivers/net/ethernet/intel/ice/ice_common.c @@ -933,216 +933,6 @@ static void ice_cleanup_fltr_mgmt_struct(struct ice_hw *hw) devm_kfree(ice_hw_to_dev(hw), sw); } -/** - * ice_get_fw_log_cfg - get FW logging configuration - * @hw: pointer to the HW struct - */ -static int ice_get_fw_log_cfg(struct ice_hw *hw) -{ - struct ice_aq_desc desc; - __le16 *config; - int status; - u16 size; - - size = sizeof(*config) * ICE_AQC_FW_LOG_ID_MAX; - config = kzalloc(size, GFP_KERNEL); - if (!config) - return -ENOMEM; - - ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_fw_logging_info); - - status = ice_aq_send_cmd(hw, &desc, config, size, NULL); - if (!status) { - u16 i; - - /* Save FW logging information into the HW structure */ - for (i = 0; i < ICE_AQC_FW_LOG_ID_MAX; i++) { - u16 v, m, flgs; - - v = le16_to_cpu(config[i]); - m = (v & ICE_AQC_FW_LOG_ID_M) >> ICE_AQC_FW_LOG_ID_S; - flgs = (v & ICE_AQC_FW_LOG_EN_M) >> ICE_AQC_FW_LOG_EN_S; - - if (m < ICE_AQC_FW_LOG_ID_MAX) - hw->fw_log.evnts[m].cur = flgs; - } - } - - kfree(config); - - return status; -} - -/** - * ice_cfg_fw_log - configure FW logging - * @hw: pointer to the HW struct - * @enable: enable certain FW logging events if true, disable all if false - * - * This function enables/disables the FW logging via Rx CQ events and a UART - * port based on predetermined configurations. FW logging via the Rx CQ can be - * enabled/disabled for individual PF's. However, FW logging via the UART can - * only be enabled/disabled for all PFs on the same device. - * - * To enable overall FW logging, the "cq_en" and "uart_en" enable bits in - * hw->fw_log need to be set accordingly, e.g. based on user-provided input, - * before initializing the device. - * - * When re/configuring FW logging, callers need to update the "cfg" elements of - * the hw->fw_log.evnts array with the desired logging event configurations for - * modules of interest. When disabling FW logging completely, the callers can - * just pass false in the "enable" parameter. On completion, the function will - * update the "cur" element of the hw->fw_log.evnts array with the resulting - * logging event configurations of the modules that are being re/configured. FW - * logging modules that are not part of a reconfiguration operation retain their - * previous states. - * - * Before resetting the device, it is recommended that the driver disables FW - * logging before shutting down the control queue. When disabling FW logging - * ("enable" = false), the latest configurations of FW logging events stored in - * hw->fw_log.evnts[] are not overridden to allow them to be reconfigured after - * a device reset. - * - * When enabling FW logging to emit log messages via the Rx CQ during the - * device's initialization phase, a mechanism alternative to interrupt handlers - * needs to be used to extract FW log messages from the Rx CQ periodically and - * to prevent the Rx CQ from being full and stalling other types of control - * messages from FW to SW. Interrupts are typically disabled during the device's - * initialization phase. - */ -static int ice_cfg_fw_log(struct ice_hw *hw, bool enable) -{ - struct ice_aqc_fw_logging *cmd; - u16 i, chgs = 0, len = 0; - struct ice_aq_desc desc; - __le16 *data = NULL; - u8 actv_evnts = 0; - void *buf = NULL; - int status = 0; - - if (!hw->fw_log.cq_en && !hw->fw_log.uart_en) - return 0; - - /* Disable FW logging only when the control queue is still responsive */ - if (!enable && - (!hw->fw_log.actv_evnts || !ice_check_sq_alive(hw, &hw->adminq))) - return 0; - - /* Get current FW log settings */ - status = ice_get_fw_log_cfg(hw); - if (status) - return status; - - ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_fw_logging); - cmd = &desc.params.fw_logging; - - /* Indicate which controls are valid */ - if (hw->fw_log.cq_en) - cmd->log_ctrl_valid |= ICE_AQC_FW_LOG_AQ_VALID; - - if (hw->fw_log.uart_en) - cmd->log_ctrl_valid |= ICE_AQC_FW_LOG_UART_VALID; - - if (enable) { - /* Fill in an array of entries with FW logging modules and - * logging events being reconfigured. - */ - for (i = 0; i < ICE_AQC_FW_LOG_ID_MAX; i++) { - u16 val; - - /* Keep track of enabled event types */ - actv_evnts |= hw->fw_log.evnts[i].cfg; - - if (hw->fw_log.evnts[i].cfg == hw->fw_log.evnts[i].cur) - continue; - - if (!data) { - data = devm_kcalloc(ice_hw_to_dev(hw), - ICE_AQC_FW_LOG_ID_MAX, - sizeof(*data), - GFP_KERNEL); - if (!data) - return -ENOMEM; - } - - val = i << ICE_AQC_FW_LOG_ID_S; - val |= hw->fw_log.evnts[i].cfg << ICE_AQC_FW_LOG_EN_S; - data[chgs++] = cpu_to_le16(val); - } - - /* Only enable FW logging if at least one module is specified. - * If FW logging is currently enabled but all modules are not - * enabled to emit log messages, disable FW logging altogether. - */ - if (actv_evnts) { - /* Leave if there is effectively no change */ - if (!chgs) - goto out; - - if (hw->fw_log.cq_en) - cmd->log_ctrl |= ICE_AQC_FW_LOG_AQ_EN; - - if (hw->fw_log.uart_en) - cmd->log_ctrl |= ICE_AQC_FW_LOG_UART_EN; - - buf = data; - len = sizeof(*data) * chgs; - desc.flags |= cpu_to_le16(ICE_AQ_FLAG_RD); - } - } - - status = ice_aq_send_cmd(hw, &desc, buf, len, NULL); - if (!status) { - /* Update the current configuration to reflect events enabled. - * hw->fw_log.cq_en and hw->fw_log.uart_en indicate if the FW - * logging mode is enabled for the device. They do not reflect - * actual modules being enabled to emit log messages. So, their - * values remain unchanged even when all modules are disabled. - */ - u16 cnt = enable ? chgs : (u16)ICE_AQC_FW_LOG_ID_MAX; - - hw->fw_log.actv_evnts = actv_evnts; - for (i = 0; i < cnt; i++) { - u16 v, m; - - if (!enable) { - /* When disabling all FW logging events as part - * of device's de-initialization, the original - * configurations are retained, and can be used - * to reconfigure FW logging later if the device - * is re-initialized. - */ - hw->fw_log.evnts[i].cur = 0; - continue; - } - - v = le16_to_cpu(data[i]); - m = (v & ICE_AQC_FW_LOG_ID_M) >> ICE_AQC_FW_LOG_ID_S; - hw->fw_log.evnts[m].cur = hw->fw_log.evnts[m].cfg; - } - } - -out: - devm_kfree(ice_hw_to_dev(hw), data); - - return status; -} - -/** - * ice_output_fw_log - * @hw: pointer to the HW struct - * @desc: pointer to the AQ message descriptor - * @buf: pointer to the buffer accompanying the AQ message - * - * Formats a FW Log message and outputs it via the standard driver logs. - */ -void ice_output_fw_log(struct ice_hw *hw, struct ice_aq_desc *desc, void *buf) -{ - ice_debug(hw, ICE_DBG_FW_LOG, "[ FW Log Msg Start ]\n"); - ice_debug_array(hw, ICE_DBG_FW_LOG, 16, 1, (u8 *)buf, - le16_to_cpu(desc->datalen)); - ice_debug(hw, ICE_DBG_FW_LOG, "[ FW Log Msg End ]\n"); -} - /** * ice_get_itr_intrl_gran * @hw: pointer to the HW struct @@ -1200,10 +990,10 @@ int ice_init_hw(struct ice_hw *hw) if (status) goto err_unroll_cqinit; - /* Enable FW logging. Not fatal if this fails. */ - status = ice_cfg_fw_log(hw, true); + status = ice_fwlog_init(hw); if (status) - ice_debug(hw, ICE_DBG_INIT, "Failed to enable FW logging.\n"); + ice_debug(hw, ICE_DBG_FW_LOG, "Error initializing FW logging: %d\n", + status); status = ice_clear_pf_cfg(hw); if (status) @@ -1354,8 +1144,7 @@ void ice_deinit_hw(struct ice_hw *hw) ice_free_hw_tbls(hw); mutex_destroy(&hw->tnl_lock); - /* Attempt to disable FW logging before shutting down control queues */ - ice_cfg_fw_log(hw, false); + ice_fwlog_deinit(hw); ice_destroy_all_ctrlq(hw); /* Clear VSI contexts if not already cleared */ diff --git a/drivers/net/ethernet/intel/ice/ice_common.h b/drivers/net/ethernet/intel/ice/ice_common.h index 77b4c68cf69693..3e933f75e948ba 100644 --- a/drivers/net/ethernet/intel/ice/ice_common.h +++ b/drivers/net/ethernet/intel/ice/ice_common.h @@ -200,7 +200,6 @@ ice_aq_cfg_lan_txq(struct ice_hw *hw, struct ice_aqc_cfg_txqs_buf *buf, struct ice_sq_cd *cd); int ice_replay_vsi(struct ice_hw *hw, u16 vsi_handle); void ice_replay_post(struct ice_hw *hw); -void ice_output_fw_log(struct ice_hw *hw, struct ice_aq_desc *desc, void *buf); struct ice_q_ctx * ice_get_lan_q_ctx(struct ice_hw *hw, u16 vsi_handle, u8 tc, u16 q_handle); int ice_sbq_rw_reg(struct ice_hw *hw, struct ice_sbq_msg_input *in); diff --git a/drivers/net/ethernet/intel/ice/ice_debugfs.c b/drivers/net/ethernet/intel/ice/ice_debugfs.c new file mode 100644 index 00000000000000..c2bfba6b9ead6c --- /dev/null +++ b/drivers/net/ethernet/intel/ice/ice_debugfs.c @@ -0,0 +1,667 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2022, Intel Corporation. */ + +#include +#include +#include +#include +#include "ice.h" + +static struct dentry *ice_debugfs_root; + +/* create a define that has an extra module that doesn't really exist. this + * is so we can add a module 'all' to easily enable/disable all the modules + */ +#define ICE_NR_FW_LOG_MODULES (ICE_AQC_FW_LOG_ID_MAX + 1) + +/* the ordering in this array is important. it matches the ordering of the + * values in the FW so the index is the same value as in ice_aqc_fw_logging_mod + */ +static const char * const ice_fwlog_module_string[] = { + "general", + "ctrl", + "link", + "link_topo", + "dnl", + "i2c", + "sdp", + "mdio", + "adminq", + "hdma", + "lldp", + "dcbx", + "dcb", + "xlr", + "nvm", + "auth", + "vpd", + "iosf", + "parser", + "sw", + "scheduler", + "txq", + "rsvd", + "post", + "watchdog", + "task_dispatch", + "mng", + "synce", + "health", + "tsdrv", + "pfreg", + "mdlver", + "all", +}; + +/* the ordering in this array is important. it matches the ordering of the + * values in the FW so the index is the same value as in ice_fwlog_level + */ +static const char * const ice_fwlog_level_string[] = { + "none", + "error", + "warning", + "normal", + "verbose", +}; + +/* the order in this array is important. it matches the ordering of the + * values in the FW so the index is the same value as in ice_fwlog_level + */ +static const char * const ice_fwlog_log_size[] = { + "128K", + "256K", + "512K", + "1M", + "2M", +}; + +/** + * ice_fwlog_print_module_cfg - print current FW logging module configuration + * @hw: pointer to the HW structure + * @module: module to print + * @s: the seq file to put data into + */ +static void +ice_fwlog_print_module_cfg(struct ice_hw *hw, int module, struct seq_file *s) +{ + struct ice_fwlog_cfg *cfg = &hw->fwlog_cfg; + struct ice_fwlog_module_entry *entry; + + if (module != ICE_AQC_FW_LOG_ID_MAX) { + entry = &cfg->module_entries[module]; + + seq_printf(s, "\tModule: %s, Log Level: %s\n", + ice_fwlog_module_string[entry->module_id], + ice_fwlog_level_string[entry->log_level]); + } else { + int i; + + for (i = 0; i < ICE_AQC_FW_LOG_ID_MAX; i++) { + entry = &cfg->module_entries[i]; + + seq_printf(s, "\tModule: %s, Log Level: %s\n", + ice_fwlog_module_string[entry->module_id], + ice_fwlog_level_string[entry->log_level]); + } + } +} + +static int ice_find_module_by_dentry(struct ice_pf *pf, struct dentry *d) +{ + int i, module; + + module = -1; + /* find the module based on the dentry */ + for (i = 0; i < ICE_NR_FW_LOG_MODULES; i++) { + if (d == pf->ice_debugfs_pf_fwlog_modules[i]) { + module = i; + break; + } + } + + return module; +} + +/** + * ice_debugfs_module_show - read from 'module' file + * @s: the opened file + * @v: pointer to the offset + */ +static int ice_debugfs_module_show(struct seq_file *s, void *v) +{ + const struct file *filp = s->file; + struct dentry *dentry; + struct ice_pf *pf; + int module; + + dentry = file_dentry(filp); + pf = s->private; + + module = ice_find_module_by_dentry(pf, dentry); + if (module < 0) { + dev_info(ice_pf_to_dev(pf), "unknown module\n"); + return -EINVAL; + } + + ice_fwlog_print_module_cfg(&pf->hw, module, s); + + return 0; +} + +static int ice_debugfs_module_open(struct inode *inode, struct file *filp) +{ + return single_open(filp, ice_debugfs_module_show, inode->i_private); +} + +/** + * ice_debugfs_module_write - write into 'module' file + * @filp: the opened file + * @buf: where to find the user's data + * @count: the length of the user's data + * @ppos: file position offset + */ +static ssize_t +ice_debugfs_module_write(struct file *filp, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct ice_pf *pf = file_inode(filp)->i_private; + struct dentry *dentry = file_dentry(filp); + struct device *dev = ice_pf_to_dev(pf); + char user_val[16], *cmd_buf; + int module, log_level, cnt; + + /* don't allow partial writes or invalid input */ + if (*ppos != 0 || count > 8) + return -EINVAL; + + cmd_buf = memdup_user(buf, count); + if (IS_ERR(cmd_buf)) + return PTR_ERR(cmd_buf); + + module = ice_find_module_by_dentry(pf, dentry); + if (module < 0) { + dev_info(dev, "unknown module\n"); + return -EINVAL; + } + + cnt = sscanf(cmd_buf, "%s", user_val); + if (cnt != 1) + return -EINVAL; + + log_level = sysfs_match_string(ice_fwlog_level_string, user_val); + if (log_level < 0) { + dev_info(dev, "unknown log level '%s'\n", user_val); + return -EINVAL; + } + + if (module != ICE_AQC_FW_LOG_ID_MAX) { + ice_pf_fwlog_update_module(pf, log_level, module); + } else { + /* the module 'all' is a shortcut so that we can set + * all of the modules to the same level quickly + */ + int i; + + for (i = 0; i < ICE_AQC_FW_LOG_ID_MAX; i++) + ice_pf_fwlog_update_module(pf, log_level, i); + } + + return count; +} + +static const struct file_operations ice_debugfs_module_fops = { + .owner = THIS_MODULE, + .open = ice_debugfs_module_open, + .read = seq_read, + .release = single_release, + .write = ice_debugfs_module_write, +}; + +/** + * ice_debugfs_nr_messages_read - read from 'nr_messages' file + * @filp: the opened file + * @buffer: where to write the data for the user to read + * @count: the size of the user's buffer + * @ppos: file position offset + */ +static ssize_t ice_debugfs_nr_messages_read(struct file *filp, + char __user *buffer, size_t count, + loff_t *ppos) +{ + struct ice_pf *pf = filp->private_data; + struct ice_hw *hw = &pf->hw; + char buff[32] = {}; + + snprintf(buff, sizeof(buff), "%d\n", + hw->fwlog_cfg.log_resolution); + + return simple_read_from_buffer(buffer, count, ppos, buff, strlen(buff)); +} + +/** + * ice_debugfs_nr_messages_write - write into 'nr_messages' file + * @filp: the opened file + * @buf: where to find the user's data + * @count: the length of the user's data + * @ppos: file position offset + */ +static ssize_t +ice_debugfs_nr_messages_write(struct file *filp, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct ice_pf *pf = filp->private_data; + struct device *dev = ice_pf_to_dev(pf); + struct ice_hw *hw = &pf->hw; + char user_val[8], *cmd_buf; + s16 nr_messages; + ssize_t ret; + + /* don't allow partial writes or invalid input */ + if (*ppos != 0 || count > 4) + return -EINVAL; + + cmd_buf = memdup_user(buf, count); + if (IS_ERR(cmd_buf)) + return PTR_ERR(cmd_buf); + + ret = sscanf(cmd_buf, "%s", user_val); + if (ret != 1) + return -EINVAL; + + ret = kstrtos16(user_val, 0, &nr_messages); + if (ret) + return ret; + + if (nr_messages < ICE_AQC_FW_LOG_MIN_RESOLUTION || + nr_messages > ICE_AQC_FW_LOG_MAX_RESOLUTION) { + dev_err(dev, "Invalid FW log number of messages %d, value must be between %d - %d\n", + nr_messages, ICE_AQC_FW_LOG_MIN_RESOLUTION, + ICE_AQC_FW_LOG_MAX_RESOLUTION); + return -EINVAL; + } + + hw->fwlog_cfg.log_resolution = nr_messages; + + return count; +} + +static const struct file_operations ice_debugfs_nr_messages_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = ice_debugfs_nr_messages_read, + .write = ice_debugfs_nr_messages_write, +}; + +/** + * ice_debugfs_enable_read - read from 'enable' file + * @filp: the opened file + * @buffer: where to write the data for the user to read + * @count: the size of the user's buffer + * @ppos: file position offset + */ +static ssize_t ice_debugfs_enable_read(struct file *filp, + char __user *buffer, size_t count, + loff_t *ppos) +{ + struct ice_pf *pf = filp->private_data; + struct ice_hw *hw = &pf->hw; + char buff[32] = {}; + + snprintf(buff, sizeof(buff), "%u\n", + (u16)(hw->fwlog_cfg.options & + ICE_FWLOG_OPTION_IS_REGISTERED) >> 3); + + return simple_read_from_buffer(buffer, count, ppos, buff, strlen(buff)); +} + +/** + * ice_debugfs_enable_write - write into 'enable' file + * @filp: the opened file + * @buf: where to find the user's data + * @count: the length of the user's data + * @ppos: file position offset + */ +static ssize_t +ice_debugfs_enable_write(struct file *filp, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct ice_pf *pf = filp->private_data; + struct ice_hw *hw = &pf->hw; + char user_val[8], *cmd_buf; + bool enable; + ssize_t ret; + + /* don't allow partial writes or invalid input */ + if (*ppos != 0 || count > 2) + return -EINVAL; + + cmd_buf = memdup_user(buf, count); + if (IS_ERR(cmd_buf)) + return PTR_ERR(cmd_buf); + + ret = sscanf(cmd_buf, "%s", user_val); + if (ret != 1) + return -EINVAL; + + ret = kstrtobool(user_val, &enable); + if (ret) + goto enable_write_error; + + if (enable) + hw->fwlog_cfg.options |= ICE_FWLOG_OPTION_ARQ_ENA; + else + hw->fwlog_cfg.options &= ~ICE_FWLOG_OPTION_ARQ_ENA; + + ret = ice_fwlog_set(hw, &hw->fwlog_cfg); + if (ret) + goto enable_write_error; + + if (enable) + ret = ice_fwlog_register(hw); + else + ret = ice_fwlog_unregister(hw); + + if (ret) + goto enable_write_error; + + /* if we get here, nothing went wrong; return count since we didn't + * really write anything + */ + ret = (ssize_t)count; + +enable_write_error: + /* This function always consumes all of the written input, or produces + * an error. Check and enforce this. Otherwise, the write operation + * won't complete properly. + */ + if (WARN_ON(ret != (ssize_t)count && ret >= 0)) + ret = -EIO; + + return ret; +} + +static const struct file_operations ice_debugfs_enable_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = ice_debugfs_enable_read, + .write = ice_debugfs_enable_write, +}; + +/** + * ice_debugfs_log_size_read - read from 'log_size' file + * @filp: the opened file + * @buffer: where to write the data for the user to read + * @count: the size of the user's buffer + * @ppos: file position offset + */ +static ssize_t ice_debugfs_log_size_read(struct file *filp, + char __user *buffer, size_t count, + loff_t *ppos) +{ + struct ice_pf *pf = filp->private_data; + struct ice_hw *hw = &pf->hw; + char buff[32] = {}; + int index; + + index = hw->fwlog_ring.index; + snprintf(buff, sizeof(buff), "%s\n", ice_fwlog_log_size[index]); + + return simple_read_from_buffer(buffer, count, ppos, buff, strlen(buff)); +} + +/** + * ice_debugfs_log_size_write - write into 'log_size' file + * @filp: the opened file + * @buf: where to find the user's data + * @count: the length of the user's data + * @ppos: file position offset + */ +static ssize_t +ice_debugfs_log_size_write(struct file *filp, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct ice_pf *pf = filp->private_data; + struct device *dev = ice_pf_to_dev(pf); + struct ice_hw *hw = &pf->hw; + char user_val[8], *cmd_buf; + ssize_t ret; + int index; + + /* don't allow partial writes or invalid input */ + if (*ppos != 0 || count > 5) + return -EINVAL; + + cmd_buf = memdup_user(buf, count); + if (IS_ERR(cmd_buf)) + return PTR_ERR(cmd_buf); + + ret = sscanf(cmd_buf, "%s", user_val); + if (ret != 1) + return -EINVAL; + + index = sysfs_match_string(ice_fwlog_log_size, user_val); + if (index < 0) { + dev_info(dev, "Invalid log size '%s'. The value must be one of 128K, 256K, 512K, 1M, 2M\n", + user_val); + ret = -EINVAL; + goto log_size_write_error; + } else if (hw->fwlog_cfg.options & ICE_FWLOG_OPTION_IS_REGISTERED) { + dev_info(dev, "FW logging is currently running. Please disable FW logging to change log_size\n"); + ret = -EINVAL; + goto log_size_write_error; + } + + /* free all the buffers and the tracking info and resize */ + ice_fwlog_realloc_rings(hw, index); + + /* if we get here, nothing went wrong; return count since we didn't + * really write anything + */ + ret = (ssize_t)count; + +log_size_write_error: + /* This function always consumes all of the written input, or produces + * an error. Check and enforce this. Otherwise, the write operation + * won't complete properly. + */ + if (WARN_ON(ret != (ssize_t)count && ret >= 0)) + ret = -EIO; + + return ret; +} + +static const struct file_operations ice_debugfs_log_size_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = ice_debugfs_log_size_read, + .write = ice_debugfs_log_size_write, +}; + +/** + * ice_debugfs_data_read - read from 'data' file + * @filp: the opened file + * @buffer: where to write the data for the user to read + * @count: the size of the user's buffer + * @ppos: file position offset + */ +static ssize_t ice_debugfs_data_read(struct file *filp, char __user *buffer, + size_t count, loff_t *ppos) +{ + struct ice_pf *pf = filp->private_data; + struct ice_hw *hw = &pf->hw; + int data_copied = 0; + bool done = false; + + if (ice_fwlog_ring_empty(&hw->fwlog_ring)) + return 0; + + while (!ice_fwlog_ring_empty(&hw->fwlog_ring) && !done) { + struct ice_fwlog_data *log; + u16 cur_buf_len; + + log = &hw->fwlog_ring.rings[hw->fwlog_ring.head]; + cur_buf_len = log->data_size; + if (cur_buf_len >= count) { + done = true; + continue; + } + + if (copy_to_user(buffer, log->data, cur_buf_len)) { + /* if there is an error then bail and return whatever + * the driver has copied so far + */ + done = true; + continue; + } + + data_copied += cur_buf_len; + buffer += cur_buf_len; + count -= cur_buf_len; + *ppos += cur_buf_len; + ice_fwlog_ring_increment(&hw->fwlog_ring.head, + hw->fwlog_ring.size); + } + + return data_copied; +} + +/** + * ice_debugfs_data_write - write into 'data' file + * @filp: the opened file + * @buf: where to find the user's data + * @count: the length of the user's data + * @ppos: file position offset + */ +static ssize_t +ice_debugfs_data_write(struct file *filp, const char __user *buf, size_t count, + loff_t *ppos) +{ + struct ice_pf *pf = filp->private_data; + struct device *dev = ice_pf_to_dev(pf); + struct ice_hw *hw = &pf->hw; + ssize_t ret; + + /* don't allow partial writes */ + if (*ppos != 0) + return 0; + + /* any value is allowed to clear the buffer so no need to even look at + * what the value is + */ + if (!(hw->fwlog_cfg.options & ICE_FWLOG_OPTION_IS_REGISTERED)) { + hw->fwlog_ring.head = 0; + hw->fwlog_ring.tail = 0; + } else { + dev_info(dev, "Can't clear FW log data while FW log running\n"); + ret = -EINVAL; + goto nr_buffs_write_error; + } + + /* if we get here, nothing went wrong; return count since we didn't + * really write anything + */ + ret = (ssize_t)count; + +nr_buffs_write_error: + /* This function always consumes all of the written input, or produces + * an error. Check and enforce this. Otherwise, the write operation + * won't complete properly. + */ + if (WARN_ON(ret != (ssize_t)count && ret >= 0)) + ret = -EIO; + + return ret; +} + +static const struct file_operations ice_debugfs_data_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = ice_debugfs_data_read, + .write = ice_debugfs_data_write, +}; + +/** + * ice_debugfs_fwlog_init - setup the debugfs directory + * @pf: the ice that is starting up + */ +void ice_debugfs_fwlog_init(struct ice_pf *pf) +{ + const char *name = pci_name(pf->pdev); + struct dentry *fw_modules_dir; + struct dentry **fw_modules; + int i; + + /* only support fw log commands on PF 0 */ + if (pf->hw.bus.func) + return; + + /* allocate space for this first because if it fails then we don't + * need to unwind + */ + fw_modules = kcalloc(ICE_NR_FW_LOG_MODULES, sizeof(*fw_modules), + GFP_KERNEL); + if (!fw_modules) + return; + + pf->ice_debugfs_pf = debugfs_create_dir(name, ice_debugfs_root); + if (IS_ERR(pf->ice_debugfs_pf)) + goto err_create_module_files; + + pf->ice_debugfs_pf_fwlog = debugfs_create_dir("fwlog", + pf->ice_debugfs_pf); + if (IS_ERR(pf->ice_debugfs_pf)) + goto err_create_module_files; + + fw_modules_dir = debugfs_create_dir("modules", + pf->ice_debugfs_pf_fwlog); + if (IS_ERR(fw_modules_dir)) + goto err_create_module_files; + + for (i = 0; i < ICE_NR_FW_LOG_MODULES; i++) { + fw_modules[i] = debugfs_create_file(ice_fwlog_module_string[i], + 0600, fw_modules_dir, pf, + &ice_debugfs_module_fops); + if (IS_ERR(fw_modules[i])) + goto err_create_module_files; + } + + debugfs_create_file("nr_messages", 0600, + pf->ice_debugfs_pf_fwlog, pf, + &ice_debugfs_nr_messages_fops); + + pf->ice_debugfs_pf_fwlog_modules = fw_modules; + + debugfs_create_file("enable", 0600, pf->ice_debugfs_pf_fwlog, + pf, &ice_debugfs_enable_fops); + + debugfs_create_file("log_size", 0600, pf->ice_debugfs_pf_fwlog, + pf, &ice_debugfs_log_size_fops); + + debugfs_create_file("data", 0600, pf->ice_debugfs_pf_fwlog, + pf, &ice_debugfs_data_fops); + + return; + +err_create_module_files: + debugfs_remove_recursive(pf->ice_debugfs_pf_fwlog); + kfree(fw_modules); +} + +/** + * ice_debugfs_init - create root directory for debugfs entries + */ +void ice_debugfs_init(void) +{ + ice_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL); + if (IS_ERR(ice_debugfs_root)) + pr_info("init of debugfs failed\n"); +} + +/** + * ice_debugfs_exit - remove debugfs entries + */ +void ice_debugfs_exit(void) +{ + debugfs_remove_recursive(ice_debugfs_root); + ice_debugfs_root = NULL; +} diff --git a/drivers/net/ethernet/intel/ice/ice_fwlog.c b/drivers/net/ethernet/intel/ice/ice_fwlog.c new file mode 100644 index 00000000000000..92b5dac481cdec --- /dev/null +++ b/drivers/net/ethernet/intel/ice/ice_fwlog.c @@ -0,0 +1,470 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2022, Intel Corporation. */ + +#include +#include "ice.h" +#include "ice_common.h" +#include "ice_fwlog.h" + +bool ice_fwlog_ring_full(struct ice_fwlog_ring *rings) +{ + u16 head, tail; + + head = rings->head; + tail = rings->tail; + + if (head < tail && (tail - head == (rings->size - 1))) + return true; + else if (head > tail && (tail == (head - 1))) + return true; + + return false; +} + +bool ice_fwlog_ring_empty(struct ice_fwlog_ring *rings) +{ + return rings->head == rings->tail; +} + +void ice_fwlog_ring_increment(u16 *item, u16 size) +{ + *item = (*item + 1) & (size - 1); +} + +static int ice_fwlog_alloc_ring_buffs(struct ice_fwlog_ring *rings) +{ + int i, nr_bytes; + u8 *mem; + + nr_bytes = rings->size * ICE_AQ_MAX_BUF_LEN; + mem = vzalloc(nr_bytes); + if (!mem) + return -ENOMEM; + + for (i = 0; i < rings->size; i++) { + struct ice_fwlog_data *ring = &rings->rings[i]; + + ring->data_size = ICE_AQ_MAX_BUF_LEN; + ring->data = mem; + mem += ICE_AQ_MAX_BUF_LEN; + } + + return 0; +} + +static void ice_fwlog_free_ring_buffs(struct ice_fwlog_ring *rings) +{ + int i; + + for (i = 0; i < rings->size; i++) { + struct ice_fwlog_data *ring = &rings->rings[i]; + + /* the first ring is the base memory for the whole range so + * free it + */ + if (!i) + vfree(ring->data); + + ring->data = NULL; + ring->data_size = 0; + } +} + +#define ICE_FWLOG_INDEX_TO_BYTES(n) ((128 * 1024) << (n)) +/** + * ice_fwlog_realloc_rings - reallocate the FW log rings + * @hw: pointer to the HW structure + * @index: the new index to use to allocate memory for the log data + * + */ +void ice_fwlog_realloc_rings(struct ice_hw *hw, int index) +{ + struct ice_fwlog_ring ring; + int status, ring_size; + + /* convert the number of bytes into a number of 4K buffers. externally + * the driver presents the interface to the FW log data as a number of + * bytes because that's easy for users to understand. internally the + * driver uses a ring of buffers because the driver doesn't know where + * the beginning and end of any line of log data is so the driver has + * to overwrite data as complete blocks. when the data is returned to + * the user the driver knows that the data is correct and the FW log + * can be correctly parsed by the tools + */ + ring_size = ICE_FWLOG_INDEX_TO_BYTES(index) / ICE_AQ_MAX_BUF_LEN; + if (ring_size == hw->fwlog_ring.size) + return; + + /* allocate space for the new rings and buffers then release the + * old rings and buffers. that way if we don't have enough + * memory then we at least have what we had before + */ + ring.rings = kcalloc(ring_size, sizeof(*ring.rings), GFP_KERNEL); + if (!ring.rings) + return; + + ring.size = ring_size; + + status = ice_fwlog_alloc_ring_buffs(&ring); + if (status) { + dev_warn(ice_hw_to_dev(hw), "Unable to allocate memory for FW log ring data buffers\n"); + ice_fwlog_free_ring_buffs(&ring); + kfree(ring.rings); + return; + } + + ice_fwlog_free_ring_buffs(&hw->fwlog_ring); + kfree(hw->fwlog_ring.rings); + + hw->fwlog_ring.rings = ring.rings; + hw->fwlog_ring.size = ring.size; + hw->fwlog_ring.index = index; + hw->fwlog_ring.head = 0; + hw->fwlog_ring.tail = 0; +} + +/** + * ice_fwlog_init - Initialize FW logging configuration + * @hw: pointer to the HW structure + * + * This function should be called on driver initialization during + * ice_init_hw(). + */ +int ice_fwlog_init(struct ice_hw *hw) +{ + /* only support fw log commands on PF 0 */ + if (hw->bus.func) + return -EINVAL; + + ice_fwlog_set_supported(hw); + + if (ice_fwlog_supported(hw)) { + int status; + + /* read the current config from the FW and store it */ + status = ice_fwlog_get(hw, &hw->fwlog_cfg); + if (status) + return status; + + hw->fwlog_ring.rings = kcalloc(ICE_FWLOG_RING_SIZE_DFLT, + sizeof(*hw->fwlog_ring.rings), + GFP_KERNEL); + if (!hw->fwlog_ring.rings) { + dev_warn(ice_hw_to_dev(hw), "Unable to allocate memory for FW log rings\n"); + return -ENOMEM; + } + + hw->fwlog_ring.size = ICE_FWLOG_RING_SIZE_DFLT; + hw->fwlog_ring.index = ICE_FWLOG_RING_SIZE_INDEX_DFLT; + + status = ice_fwlog_alloc_ring_buffs(&hw->fwlog_ring); + if (status) { + dev_warn(ice_hw_to_dev(hw), "Unable to allocate memory for FW log ring data buffers\n"); + ice_fwlog_free_ring_buffs(&hw->fwlog_ring); + kfree(hw->fwlog_ring.rings); + return status; + } + + ice_debugfs_fwlog_init(hw->back); + } else { + dev_warn(ice_hw_to_dev(hw), "FW logging is not supported in this NVM image. Please update the NVM to get FW log support\n"); + } + + return 0; +} + +/** + * ice_fwlog_deinit - unroll FW logging configuration + * @hw: pointer to the HW structure + * + * This function should be called in ice_deinit_hw(). + */ +void ice_fwlog_deinit(struct ice_hw *hw) +{ + struct ice_pf *pf = hw->back; + int status; + + /* only support fw log commands on PF 0 */ + if (hw->bus.func) + return; + + /* make sure FW logging is disabled to not put the FW in a weird state + * for the next driver load + */ + hw->fwlog_cfg.options &= ~ICE_FWLOG_OPTION_ARQ_ENA; + status = ice_fwlog_set(hw, &hw->fwlog_cfg); + if (status) + dev_warn(ice_hw_to_dev(hw), "Unable to turn off FW logging, status: %d\n", + status); + + kfree(pf->ice_debugfs_pf_fwlog_modules); + + pf->ice_debugfs_pf_fwlog_modules = NULL; + + status = ice_fwlog_unregister(hw); + if (status) + dev_warn(ice_hw_to_dev(hw), "Unable to unregister FW logging, status: %d\n", + status); + + if (hw->fwlog_ring.rings) { + ice_fwlog_free_ring_buffs(&hw->fwlog_ring); + kfree(hw->fwlog_ring.rings); + } +} + +/** + * ice_fwlog_supported - Cached for whether FW supports FW logging or not + * @hw: pointer to the HW structure + * + * This will always return false if called before ice_init_hw(), so it must be + * called after ice_init_hw(). + */ +bool ice_fwlog_supported(struct ice_hw *hw) +{ + return hw->fwlog_supported; +} + +/** + * ice_aq_fwlog_set - Set FW logging configuration AQ command (0xFF30) + * @hw: pointer to the HW structure + * @entries: entries to configure + * @num_entries: number of @entries + * @options: options from ice_fwlog_cfg->options structure + * @log_resolution: logging resolution + */ +static int +ice_aq_fwlog_set(struct ice_hw *hw, struct ice_fwlog_module_entry *entries, + u16 num_entries, u16 options, u16 log_resolution) +{ + struct ice_aqc_fw_log_cfg_resp *fw_modules; + struct ice_aqc_fw_log *cmd; + struct ice_aq_desc desc; + int status; + int i; + + fw_modules = kcalloc(num_entries, sizeof(*fw_modules), GFP_KERNEL); + if (!fw_modules) + return -ENOMEM; + + for (i = 0; i < num_entries; i++) { + fw_modules[i].module_identifier = + cpu_to_le16(entries[i].module_id); + fw_modules[i].log_level = entries[i].log_level; + } + + ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_fw_logs_config); + desc.flags |= cpu_to_le16(ICE_AQ_FLAG_RD); + + cmd = &desc.params.fw_log; + + cmd->cmd_flags = ICE_AQC_FW_LOG_CONF_SET_VALID; + cmd->ops.cfg.log_resolution = cpu_to_le16(log_resolution); + cmd->ops.cfg.mdl_cnt = cpu_to_le16(num_entries); + + if (options & ICE_FWLOG_OPTION_ARQ_ENA) + cmd->cmd_flags |= ICE_AQC_FW_LOG_CONF_AQ_EN; + if (options & ICE_FWLOG_OPTION_UART_ENA) + cmd->cmd_flags |= ICE_AQC_FW_LOG_CONF_UART_EN; + + status = ice_aq_send_cmd(hw, &desc, fw_modules, + sizeof(*fw_modules) * num_entries, + NULL); + + kfree(fw_modules); + + return status; +} + +/** + * ice_fwlog_set - Set the firmware logging settings + * @hw: pointer to the HW structure + * @cfg: config used to set firmware logging + * + * This function should be called whenever the driver needs to set the firmware + * logging configuration. It can be called on initialization, reset, or during + * runtime. + * + * If the PF wishes to receive FW logging then it must register via + * ice_fwlog_register. Note, that ice_fwlog_register does not need to be called + * for init. + */ +int ice_fwlog_set(struct ice_hw *hw, struct ice_fwlog_cfg *cfg) +{ + if (!ice_fwlog_supported(hw)) + return -EOPNOTSUPP; + + return ice_aq_fwlog_set(hw, cfg->module_entries, + ICE_AQC_FW_LOG_ID_MAX, cfg->options, + cfg->log_resolution); +} + +/** + * ice_aq_fwlog_get - Get the current firmware logging configuration (0xFF32) + * @hw: pointer to the HW structure + * @cfg: firmware logging configuration to populate + */ +static int ice_aq_fwlog_get(struct ice_hw *hw, struct ice_fwlog_cfg *cfg) +{ + struct ice_aqc_fw_log_cfg_resp *fw_modules; + struct ice_aqc_fw_log *cmd; + struct ice_aq_desc desc; + u16 module_id_cnt; + int status; + void *buf; + int i; + + memset(cfg, 0, sizeof(*cfg)); + + buf = kzalloc(ICE_AQ_MAX_BUF_LEN, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_fw_logs_query); + cmd = &desc.params.fw_log; + + cmd->cmd_flags = ICE_AQC_FW_LOG_AQ_QUERY; + + status = ice_aq_send_cmd(hw, &desc, buf, ICE_AQ_MAX_BUF_LEN, NULL); + if (status) { + ice_debug(hw, ICE_DBG_FW_LOG, "Failed to get FW log configuration\n"); + goto status_out; + } + + module_id_cnt = le16_to_cpu(cmd->ops.cfg.mdl_cnt); + if (module_id_cnt < ICE_AQC_FW_LOG_ID_MAX) { + ice_debug(hw, ICE_DBG_FW_LOG, "FW returned less than the expected number of FW log module IDs\n"); + } else if (module_id_cnt > ICE_AQC_FW_LOG_ID_MAX) { + ice_debug(hw, ICE_DBG_FW_LOG, "FW returned more than expected number of FW log module IDs, setting module_id_cnt to software expected max %u\n", + ICE_AQC_FW_LOG_ID_MAX); + module_id_cnt = ICE_AQC_FW_LOG_ID_MAX; + } + + cfg->log_resolution = le16_to_cpu(cmd->ops.cfg.log_resolution); + if (cmd->cmd_flags & ICE_AQC_FW_LOG_CONF_AQ_EN) + cfg->options |= ICE_FWLOG_OPTION_ARQ_ENA; + if (cmd->cmd_flags & ICE_AQC_FW_LOG_CONF_UART_EN) + cfg->options |= ICE_FWLOG_OPTION_UART_ENA; + if (cmd->cmd_flags & ICE_AQC_FW_LOG_QUERY_REGISTERED) + cfg->options |= ICE_FWLOG_OPTION_IS_REGISTERED; + + fw_modules = (struct ice_aqc_fw_log_cfg_resp *)buf; + + for (i = 0; i < module_id_cnt; i++) { + struct ice_aqc_fw_log_cfg_resp *fw_module = &fw_modules[i]; + + cfg->module_entries[i].module_id = + le16_to_cpu(fw_module->module_identifier); + cfg->module_entries[i].log_level = fw_module->log_level; + } + +status_out: + kfree(buf); + return status; +} + +/** + * ice_fwlog_get - Get the firmware logging settings + * @hw: pointer to the HW structure + * @cfg: config to populate based on current firmware logging settings + */ +int ice_fwlog_get(struct ice_hw *hw, struct ice_fwlog_cfg *cfg) +{ + if (!ice_fwlog_supported(hw)) + return -EOPNOTSUPP; + + return ice_aq_fwlog_get(hw, cfg); +} + +/** + * ice_aq_fwlog_register - Register PF for firmware logging events (0xFF31) + * @hw: pointer to the HW structure + * @reg: true to register and false to unregister + */ +static int ice_aq_fwlog_register(struct ice_hw *hw, bool reg) +{ + struct ice_aq_desc desc; + + ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_fw_logs_register); + + if (reg) + desc.params.fw_log.cmd_flags = ICE_AQC_FW_LOG_AQ_REGISTER; + + return ice_aq_send_cmd(hw, &desc, NULL, 0, NULL); +} + +/** + * ice_fwlog_register - Register the PF for firmware logging + * @hw: pointer to the HW structure + * + * After this call the PF will start to receive firmware logging based on the + * configuration set in ice_fwlog_set. + */ +int ice_fwlog_register(struct ice_hw *hw) +{ + int status; + + if (!ice_fwlog_supported(hw)) + return -EOPNOTSUPP; + + status = ice_aq_fwlog_register(hw, true); + if (status) + ice_debug(hw, ICE_DBG_FW_LOG, "Failed to register for firmware logging events over ARQ\n"); + else + hw->fwlog_cfg.options |= ICE_FWLOG_OPTION_IS_REGISTERED; + + return status; +} + +/** + * ice_fwlog_unregister - Unregister the PF from firmware logging + * @hw: pointer to the HW structure + */ +int ice_fwlog_unregister(struct ice_hw *hw) +{ + int status; + + if (!ice_fwlog_supported(hw)) + return -EOPNOTSUPP; + + status = ice_aq_fwlog_register(hw, false); + if (status) + ice_debug(hw, ICE_DBG_FW_LOG, "Failed to unregister from firmware logging events over ARQ\n"); + else + hw->fwlog_cfg.options &= ~ICE_FWLOG_OPTION_IS_REGISTERED; + + return status; +} + +/** + * ice_fwlog_set_supported - Set if FW logging is supported by FW + * @hw: pointer to the HW struct + * + * If FW returns success to the ice_aq_fwlog_get call then it supports FW + * logging, else it doesn't. Set the fwlog_supported flag accordingly. + * + * This function is only meant to be called during driver init to determine if + * the FW support FW logging. + */ +void ice_fwlog_set_supported(struct ice_hw *hw) +{ + struct ice_fwlog_cfg *cfg; + int status; + + hw->fwlog_supported = false; + + cfg = kzalloc(sizeof(*cfg), GFP_KERNEL); + if (!cfg) + return; + + /* don't call ice_fwlog_get() because that would check to see if FW + * logging is supported which is what the driver is determining now + */ + status = ice_aq_fwlog_get(hw, cfg); + if (status) + ice_debug(hw, ICE_DBG_FW_LOG, "ice_aq_fwlog_get failed, FW logging is not supported on this version of FW, status %d\n", + status); + else + hw->fwlog_supported = true; + + kfree(cfg); +} diff --git a/drivers/net/ethernet/intel/ice/ice_fwlog.h b/drivers/net/ethernet/intel/ice/ice_fwlog.h new file mode 100644 index 00000000000000..287e71fa4b8624 --- /dev/null +++ b/drivers/net/ethernet/intel/ice/ice_fwlog.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2022, Intel Corporation. */ + +#ifndef _ICE_FWLOG_H_ +#define _ICE_FWLOG_H_ +#include "ice_adminq_cmd.h" + +struct ice_hw; + +/* Only a single log level should be set and all log levels under the set value + * are enabled, e.g. if log level is set to ICE_FW_LOG_LEVEL_VERBOSE, then all + * other log levels are included (except ICE_FW_LOG_LEVEL_NONE) + */ +enum ice_fwlog_level { + ICE_FWLOG_LEVEL_NONE = 0, + ICE_FWLOG_LEVEL_ERROR = 1, + ICE_FWLOG_LEVEL_WARNING = 2, + ICE_FWLOG_LEVEL_NORMAL = 3, + ICE_FWLOG_LEVEL_VERBOSE = 4, + ICE_FWLOG_LEVEL_INVALID, /* all values >= this entry are invalid */ +}; + +struct ice_fwlog_module_entry { + /* module ID for the corresponding firmware logging event */ + u16 module_id; + /* verbosity level for the module_id */ + u8 log_level; +}; + +struct ice_fwlog_cfg { + /* list of modules for configuring log level */ + struct ice_fwlog_module_entry module_entries[ICE_AQC_FW_LOG_ID_MAX]; + /* options used to configure firmware logging */ + u16 options; +#define ICE_FWLOG_OPTION_ARQ_ENA BIT(0) +#define ICE_FWLOG_OPTION_UART_ENA BIT(1) + /* set before calling ice_fwlog_init() so the PF registers for firmware + * logging on initialization + */ +#define ICE_FWLOG_OPTION_REGISTER_ON_INIT BIT(2) + /* set in the ice_fwlog_get() response if the PF is registered for FW + * logging events over ARQ + */ +#define ICE_FWLOG_OPTION_IS_REGISTERED BIT(3) + + /* minimum number of log events sent per Admin Receive Queue event */ + u16 log_resolution; +}; + +struct ice_fwlog_data { + u16 data_size; + u8 *data; +}; + +struct ice_fwlog_ring { + struct ice_fwlog_data *rings; + u16 index; + u16 size; + u16 head; + u16 tail; +}; + +#define ICE_FWLOG_RING_SIZE_INDEX_DFLT 3 +#define ICE_FWLOG_RING_SIZE_DFLT 256 +#define ICE_FWLOG_RING_SIZE_MAX 512 + +bool ice_fwlog_ring_full(struct ice_fwlog_ring *rings); +bool ice_fwlog_ring_empty(struct ice_fwlog_ring *rings); +void ice_fwlog_ring_increment(u16 *item, u16 size); +void ice_fwlog_set_supported(struct ice_hw *hw); +bool ice_fwlog_supported(struct ice_hw *hw); +int ice_fwlog_init(struct ice_hw *hw); +void ice_fwlog_deinit(struct ice_hw *hw); +int ice_fwlog_set(struct ice_hw *hw, struct ice_fwlog_cfg *cfg); +int ice_fwlog_get(struct ice_hw *hw, struct ice_fwlog_cfg *cfg); +int ice_fwlog_register(struct ice_hw *hw); +int ice_fwlog_unregister(struct ice_hw *hw); +void ice_fwlog_realloc_rings(struct ice_hw *hw, int index); +#endif /* _ICE_FWLOG_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_lan_tx_rx.h b/drivers/net/ethernet/intel/ice/ice_lan_tx_rx.h index 89f986a75cc855..d384ddfcb83e97 100644 --- a/drivers/net/ethernet/intel/ice/ice_lan_tx_rx.h +++ b/drivers/net/ethernet/intel/ice/ice_lan_tx_rx.h @@ -673,6 +673,212 @@ struct ice_tlan_ctx { * Use the enum ice_rx_l2_ptype to decode the packet type * ENDIF */ +#define ICE_PTYPES \ + /* L2 Packet types */ \ + ICE_PTT_UNUSED_ENTRY(0), \ + ICE_PTT(1, L2, NONE, NOF, NONE, NONE, NOF, NONE, PAY2), \ + ICE_PTT_UNUSED_ENTRY(2), \ + ICE_PTT_UNUSED_ENTRY(3), \ + ICE_PTT_UNUSED_ENTRY(4), \ + ICE_PTT_UNUSED_ENTRY(5), \ + ICE_PTT(6, L2, NONE, NOF, NONE, NONE, NOF, NONE, NONE), \ + ICE_PTT(7, L2, NONE, NOF, NONE, NONE, NOF, NONE, NONE), \ + ICE_PTT_UNUSED_ENTRY(8), \ + ICE_PTT_UNUSED_ENTRY(9), \ + ICE_PTT(10, L2, NONE, NOF, NONE, NONE, NOF, NONE, NONE), \ + ICE_PTT(11, L2, NONE, NOF, NONE, NONE, NOF, NONE, NONE), \ + ICE_PTT_UNUSED_ENTRY(12), \ + ICE_PTT_UNUSED_ENTRY(13), \ + ICE_PTT_UNUSED_ENTRY(14), \ + ICE_PTT_UNUSED_ENTRY(15), \ + ICE_PTT_UNUSED_ENTRY(16), \ + ICE_PTT_UNUSED_ENTRY(17), \ + ICE_PTT_UNUSED_ENTRY(18), \ + ICE_PTT_UNUSED_ENTRY(19), \ + ICE_PTT_UNUSED_ENTRY(20), \ + ICE_PTT_UNUSED_ENTRY(21), \ + \ + /* Non Tunneled IPv4 */ \ + ICE_PTT(22, IP, IPV4, FRG, NONE, NONE, NOF, NONE, PAY3), \ + ICE_PTT(23, IP, IPV4, NOF, NONE, NONE, NOF, NONE, PAY3), \ + ICE_PTT(24, IP, IPV4, NOF, NONE, NONE, NOF, UDP, PAY4), \ + ICE_PTT_UNUSED_ENTRY(25), \ + ICE_PTT(26, IP, IPV4, NOF, NONE, NONE, NOF, TCP, PAY4), \ + ICE_PTT(27, IP, IPV4, NOF, NONE, NONE, NOF, SCTP, PAY4), \ + ICE_PTT(28, IP, IPV4, NOF, NONE, NONE, NOF, ICMP, PAY4), \ + \ + /* IPv4 --> IPv4 */ \ + ICE_PTT(29, IP, IPV4, NOF, IP_IP, IPV4, FRG, NONE, PAY3), \ + ICE_PTT(30, IP, IPV4, NOF, IP_IP, IPV4, NOF, NONE, PAY3), \ + ICE_PTT(31, IP, IPV4, NOF, IP_IP, IPV4, NOF, UDP, PAY4), \ + ICE_PTT_UNUSED_ENTRY(32), \ + ICE_PTT(33, IP, IPV4, NOF, IP_IP, IPV4, NOF, TCP, PAY4), \ + ICE_PTT(34, IP, IPV4, NOF, IP_IP, IPV4, NOF, SCTP, PAY4), \ + ICE_PTT(35, IP, IPV4, NOF, IP_IP, IPV4, NOF, ICMP, PAY4), \ + \ + /* IPv4 --> IPv6 */ \ + ICE_PTT(36, IP, IPV4, NOF, IP_IP, IPV6, FRG, NONE, PAY3), \ + ICE_PTT(37, IP, IPV4, NOF, IP_IP, IPV6, NOF, NONE, PAY3), \ + ICE_PTT(38, IP, IPV4, NOF, IP_IP, IPV6, NOF, UDP, PAY4), \ + ICE_PTT_UNUSED_ENTRY(39), \ + ICE_PTT(40, IP, IPV4, NOF, IP_IP, IPV6, NOF, TCP, PAY4), \ + ICE_PTT(41, IP, IPV4, NOF, IP_IP, IPV6, NOF, SCTP, PAY4), \ + ICE_PTT(42, IP, IPV4, NOF, IP_IP, IPV6, NOF, ICMP, PAY4), \ + \ + /* IPv4 --> GRE/NAT */ \ + ICE_PTT(43, IP, IPV4, NOF, IP_GRENAT, NONE, NOF, NONE, PAY3), \ + \ + /* IPv4 --> GRE/NAT --> IPv4 */ \ + ICE_PTT(44, IP, IPV4, NOF, IP_GRENAT, IPV4, FRG, NONE, PAY3), \ + ICE_PTT(45, IP, IPV4, NOF, IP_GRENAT, IPV4, NOF, NONE, PAY3), \ + ICE_PTT(46, IP, IPV4, NOF, IP_GRENAT, IPV4, NOF, UDP, PAY4), \ + ICE_PTT_UNUSED_ENTRY(47), \ + ICE_PTT(48, IP, IPV4, NOF, IP_GRENAT, IPV4, NOF, TCP, PAY4), \ + ICE_PTT(49, IP, IPV4, NOF, IP_GRENAT, IPV4, NOF, SCTP, PAY4), \ + ICE_PTT(50, IP, IPV4, NOF, IP_GRENAT, IPV4, NOF, ICMP, PAY4), \ + \ + /* IPv4 --> GRE/NAT --> IPv6 */ \ + ICE_PTT(51, IP, IPV4, NOF, IP_GRENAT, IPV6, FRG, NONE, PAY3), \ + ICE_PTT(52, IP, IPV4, NOF, IP_GRENAT, IPV6, NOF, NONE, PAY3), \ + ICE_PTT(53, IP, IPV4, NOF, IP_GRENAT, IPV6, NOF, UDP, PAY4), \ + ICE_PTT_UNUSED_ENTRY(54), \ + ICE_PTT(55, IP, IPV4, NOF, IP_GRENAT, IPV6, NOF, TCP, PAY4), \ + ICE_PTT(56, IP, IPV4, NOF, IP_GRENAT, IPV6, NOF, SCTP, PAY4), \ + ICE_PTT(57, IP, IPV4, NOF, IP_GRENAT, IPV6, NOF, ICMP, PAY4), \ + \ + /* IPv4 --> GRE/NAT --> MAC */ \ + ICE_PTT(58, IP, IPV4, NOF, IP_GRENAT_MAC, NONE, NOF, NONE, PAY3), \ + \ + /* IPv4 --> GRE/NAT --> MAC --> IPv4 */ \ + ICE_PTT(59, IP, IPV4, NOF, IP_GRENAT_MAC, IPV4, FRG, NONE, PAY3), \ + ICE_PTT(60, IP, IPV4, NOF, IP_GRENAT_MAC, IPV4, NOF, NONE, PAY3), \ + ICE_PTT(61, IP, IPV4, NOF, IP_GRENAT_MAC, IPV4, NOF, UDP, PAY4), \ + ICE_PTT_UNUSED_ENTRY(62), \ + ICE_PTT(63, IP, IPV4, NOF, IP_GRENAT_MAC, IPV4, NOF, TCP, PAY4), \ + ICE_PTT(64, IP, IPV4, NOF, IP_GRENAT_MAC, IPV4, NOF, SCTP, PAY4), \ + ICE_PTT(65, IP, IPV4, NOF, IP_GRENAT_MAC, IPV4, NOF, ICMP, PAY4), \ + \ + /* IPv4 --> GRE/NAT -> MAC --> IPv6 */ \ + ICE_PTT(66, IP, IPV4, NOF, IP_GRENAT_MAC, IPV6, FRG, NONE, PAY3), \ + ICE_PTT(67, IP, IPV4, NOF, IP_GRENAT_MAC, IPV6, NOF, NONE, PAY3), \ + ICE_PTT(68, IP, IPV4, NOF, IP_GRENAT_MAC, IPV6, NOF, UDP, PAY4), \ + ICE_PTT_UNUSED_ENTRY(69), \ + ICE_PTT(70, IP, IPV4, NOF, IP_GRENAT_MAC, IPV6, NOF, TCP, PAY4), \ + ICE_PTT(71, IP, IPV4, NOF, IP_GRENAT_MAC, IPV6, NOF, SCTP, PAY4), \ + ICE_PTT(72, IP, IPV4, NOF, IP_GRENAT_MAC, IPV6, NOF, ICMP, PAY4), \ + \ + /* IPv4 --> GRE/NAT --> MAC/VLAN */ \ + ICE_PTT(73, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, NONE, NOF, NONE, PAY3), \ + \ + /* IPv4 ---> GRE/NAT -> MAC/VLAN --> IPv4 */ \ + ICE_PTT(74, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, IPV4, FRG, NONE, PAY3), \ + ICE_PTT(75, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, IPV4, NOF, NONE, PAY3), \ + ICE_PTT(76, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, IPV4, NOF, UDP, PAY4), \ + ICE_PTT_UNUSED_ENTRY(77), \ + ICE_PTT(78, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, IPV4, NOF, TCP, PAY4), \ + ICE_PTT(79, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, IPV4, NOF, SCTP, PAY4), \ + ICE_PTT(80, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, IPV4, NOF, ICMP, PAY4), \ + \ + /* IPv4 -> GRE/NAT -> MAC/VLAN --> IPv6 */ \ + ICE_PTT(81, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, IPV6, FRG, NONE, PAY3), \ + ICE_PTT(82, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, IPV6, NOF, NONE, PAY3), \ + ICE_PTT(83, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, IPV6, NOF, UDP, PAY4), \ + ICE_PTT_UNUSED_ENTRY(84), \ + ICE_PTT(85, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, IPV6, NOF, TCP, PAY4), \ + ICE_PTT(86, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, IPV6, NOF, SCTP, PAY4), \ + ICE_PTT(87, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, IPV6, NOF, ICMP, PAY4), \ + \ + /* Non Tunneled IPv6 */ \ + ICE_PTT(88, IP, IPV6, FRG, NONE, NONE, NOF, NONE, PAY3), \ + ICE_PTT(89, IP, IPV6, NOF, NONE, NONE, NOF, NONE, PAY3), \ + ICE_PTT(90, IP, IPV6, NOF, NONE, NONE, NOF, UDP, PAY4), \ + ICE_PTT_UNUSED_ENTRY(91), \ + ICE_PTT(92, IP, IPV6, NOF, NONE, NONE, NOF, TCP, PAY4), \ + ICE_PTT(93, IP, IPV6, NOF, NONE, NONE, NOF, SCTP, PAY4), \ + ICE_PTT(94, IP, IPV6, NOF, NONE, NONE, NOF, ICMP, PAY4), \ + \ + /* IPv6 --> IPv4 */ \ + ICE_PTT(95, IP, IPV6, NOF, IP_IP, IPV4, FRG, NONE, PAY3), \ + ICE_PTT(96, IP, IPV6, NOF, IP_IP, IPV4, NOF, NONE, PAY3), \ + ICE_PTT(97, IP, IPV6, NOF, IP_IP, IPV4, NOF, UDP, PAY4), \ + ICE_PTT_UNUSED_ENTRY(98), \ + ICE_PTT(99, IP, IPV6, NOF, IP_IP, IPV4, NOF, TCP, PAY4), \ + ICE_PTT(100, IP, IPV6, NOF, IP_IP, IPV4, NOF, SCTP, PAY4), \ + ICE_PTT(101, IP, IPV6, NOF, IP_IP, IPV4, NOF, ICMP, PAY4), \ + \ + /* IPv6 --> IPv6 */ \ + ICE_PTT(102, IP, IPV6, NOF, IP_IP, IPV6, FRG, NONE, PAY3), \ + ICE_PTT(103, IP, IPV6, NOF, IP_IP, IPV6, NOF, NONE, PAY3), \ + ICE_PTT(104, IP, IPV6, NOF, IP_IP, IPV6, NOF, UDP, PAY4), \ + ICE_PTT_UNUSED_ENTRY(105), \ + ICE_PTT(106, IP, IPV6, NOF, IP_IP, IPV6, NOF, TCP, PAY4), \ + ICE_PTT(107, IP, IPV6, NOF, IP_IP, IPV6, NOF, SCTP, PAY4), \ + ICE_PTT(108, IP, IPV6, NOF, IP_IP, IPV6, NOF, ICMP, PAY4), \ + \ + /* IPv6 --> GRE/NAT */ \ + ICE_PTT(109, IP, IPV6, NOF, IP_GRENAT, NONE, NOF, NONE, PAY3), \ + \ + /* IPv6 --> GRE/NAT -> IPv4 */ \ + ICE_PTT(110, IP, IPV6, NOF, IP_GRENAT, IPV4, FRG, NONE, PAY3), \ + ICE_PTT(111, IP, IPV6, NOF, IP_GRENAT, IPV4, NOF, NONE, PAY3), \ + ICE_PTT(112, IP, IPV6, NOF, IP_GRENAT, IPV4, NOF, UDP, PAY4), \ + ICE_PTT_UNUSED_ENTRY(113), \ + ICE_PTT(114, IP, IPV6, NOF, IP_GRENAT, IPV4, NOF, TCP, PAY4), \ + ICE_PTT(115, IP, IPV6, NOF, IP_GRENAT, IPV4, NOF, SCTP, PAY4), \ + ICE_PTT(116, IP, IPV6, NOF, IP_GRENAT, IPV4, NOF, ICMP, PAY4), \ + \ + /* IPv6 --> GRE/NAT -> IPv6 */ \ + ICE_PTT(117, IP, IPV6, NOF, IP_GRENAT, IPV6, FRG, NONE, PAY3), \ + ICE_PTT(118, IP, IPV6, NOF, IP_GRENAT, IPV6, NOF, NONE, PAY3), \ + ICE_PTT(119, IP, IPV6, NOF, IP_GRENAT, IPV6, NOF, UDP, PAY4), \ + ICE_PTT_UNUSED_ENTRY(120), \ + ICE_PTT(121, IP, IPV6, NOF, IP_GRENAT, IPV6, NOF, TCP, PAY4), \ + ICE_PTT(122, IP, IPV6, NOF, IP_GRENAT, IPV6, NOF, SCTP, PAY4), \ + ICE_PTT(123, IP, IPV6, NOF, IP_GRENAT, IPV6, NOF, ICMP, PAY4), \ + \ + /* IPv6 --> GRE/NAT -> MAC */ \ + ICE_PTT(124, IP, IPV6, NOF, IP_GRENAT_MAC, NONE, NOF, NONE, PAY3), \ + \ + /* IPv6 --> GRE/NAT -> MAC -> IPv4 */ \ + ICE_PTT(125, IP, IPV6, NOF, IP_GRENAT_MAC, IPV4, FRG, NONE, PAY3), \ + ICE_PTT(126, IP, IPV6, NOF, IP_GRENAT_MAC, IPV4, NOF, NONE, PAY3), \ + ICE_PTT(127, IP, IPV6, NOF, IP_GRENAT_MAC, IPV4, NOF, UDP, PAY4), \ + ICE_PTT_UNUSED_ENTRY(128), \ + ICE_PTT(129, IP, IPV6, NOF, IP_GRENAT_MAC, IPV4, NOF, TCP, PAY4), \ + ICE_PTT(130, IP, IPV6, NOF, IP_GRENAT_MAC, IPV4, NOF, SCTP, PAY4), \ + ICE_PTT(131, IP, IPV6, NOF, IP_GRENAT_MAC, IPV4, NOF, ICMP, PAY4), \ + \ + /* IPv6 --> GRE/NAT -> MAC -> IPv6 */ \ + ICE_PTT(132, IP, IPV6, NOF, IP_GRENAT_MAC, IPV6, FRG, NONE, PAY3), \ + ICE_PTT(133, IP, IPV6, NOF, IP_GRENAT_MAC, IPV6, NOF, NONE, PAY3), \ + ICE_PTT(134, IP, IPV6, NOF, IP_GRENAT_MAC, IPV6, NOF, UDP, PAY4), \ + ICE_PTT_UNUSED_ENTRY(135), \ + ICE_PTT(136, IP, IPV6, NOF, IP_GRENAT_MAC, IPV6, NOF, TCP, PAY4), \ + ICE_PTT(137, IP, IPV6, NOF, IP_GRENAT_MAC, IPV6, NOF, SCTP, PAY4), \ + ICE_PTT(138, IP, IPV6, NOF, IP_GRENAT_MAC, IPV6, NOF, ICMP, PAY4), \ + \ + /* IPv6 --> GRE/NAT -> MAC/VLAN */ \ + ICE_PTT(139, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, NONE, NOF, NONE, PAY3), \ + \ + /* IPv6 --> GRE/NAT -> MAC/VLAN --> IPv4 */ \ + ICE_PTT(140, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV4, FRG, NONE, PAY3), \ + ICE_PTT(141, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV4, NOF, NONE, PAY3), \ + ICE_PTT(142, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV4, NOF, UDP, PAY4), \ + ICE_PTT_UNUSED_ENTRY(143), \ + ICE_PTT(144, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV4, NOF, TCP, PAY4), \ + ICE_PTT(145, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV4, NOF, SCTP, PAY4), \ + ICE_PTT(146, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV4, NOF, ICMP, PAY4), \ + \ + /* IPv6 --> GRE/NAT -> MAC/VLAN --> IPv6 */ \ + ICE_PTT(147, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV6, FRG, NONE, PAY3), \ + ICE_PTT(148, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV6, NOF, NONE, PAY3), \ + ICE_PTT(149, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV6, NOF, UDP, PAY4), \ + ICE_PTT_UNUSED_ENTRY(150), \ + ICE_PTT(151, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV6, NOF, TCP, PAY4), \ + ICE_PTT(152, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV6, NOF, SCTP, PAY4), \ + ICE_PTT(153, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV6, NOF, ICMP, PAY4), + +#define ICE_NUM_DEFINED_PTYPES 154 /* macro to make the table lines short, use explicit indexing with [PTYPE] */ #define ICE_PTT(PTYPE, OUTER_IP, OUTER_IP_VER, OUTER_FRAG, T, TE, TEF, I, PL)\ @@ -695,212 +901,10 @@ struct ice_tlan_ctx { /* Lookup table mapping in the 10-bit HW PTYPE to the bit field for decoding */ static const struct ice_rx_ptype_decoded ice_ptype_lkup[BIT(10)] = { - /* L2 Packet types */ - ICE_PTT_UNUSED_ENTRY(0), - ICE_PTT(1, L2, NONE, NOF, NONE, NONE, NOF, NONE, PAY2), - ICE_PTT_UNUSED_ENTRY(2), - ICE_PTT_UNUSED_ENTRY(3), - ICE_PTT_UNUSED_ENTRY(4), - ICE_PTT_UNUSED_ENTRY(5), - ICE_PTT(6, L2, NONE, NOF, NONE, NONE, NOF, NONE, NONE), - ICE_PTT(7, L2, NONE, NOF, NONE, NONE, NOF, NONE, NONE), - ICE_PTT_UNUSED_ENTRY(8), - ICE_PTT_UNUSED_ENTRY(9), - ICE_PTT(10, L2, NONE, NOF, NONE, NONE, NOF, NONE, NONE), - ICE_PTT(11, L2, NONE, NOF, NONE, NONE, NOF, NONE, NONE), - ICE_PTT_UNUSED_ENTRY(12), - ICE_PTT_UNUSED_ENTRY(13), - ICE_PTT_UNUSED_ENTRY(14), - ICE_PTT_UNUSED_ENTRY(15), - ICE_PTT_UNUSED_ENTRY(16), - ICE_PTT_UNUSED_ENTRY(17), - ICE_PTT_UNUSED_ENTRY(18), - ICE_PTT_UNUSED_ENTRY(19), - ICE_PTT_UNUSED_ENTRY(20), - ICE_PTT_UNUSED_ENTRY(21), - - /* Non Tunneled IPv4 */ - ICE_PTT(22, IP, IPV4, FRG, NONE, NONE, NOF, NONE, PAY3), - ICE_PTT(23, IP, IPV4, NOF, NONE, NONE, NOF, NONE, PAY3), - ICE_PTT(24, IP, IPV4, NOF, NONE, NONE, NOF, UDP, PAY4), - ICE_PTT_UNUSED_ENTRY(25), - ICE_PTT(26, IP, IPV4, NOF, NONE, NONE, NOF, TCP, PAY4), - ICE_PTT(27, IP, IPV4, NOF, NONE, NONE, NOF, SCTP, PAY4), - ICE_PTT(28, IP, IPV4, NOF, NONE, NONE, NOF, ICMP, PAY4), - - /* IPv4 --> IPv4 */ - ICE_PTT(29, IP, IPV4, NOF, IP_IP, IPV4, FRG, NONE, PAY3), - ICE_PTT(30, IP, IPV4, NOF, IP_IP, IPV4, NOF, NONE, PAY3), - ICE_PTT(31, IP, IPV4, NOF, IP_IP, IPV4, NOF, UDP, PAY4), - ICE_PTT_UNUSED_ENTRY(32), - ICE_PTT(33, IP, IPV4, NOF, IP_IP, IPV4, NOF, TCP, PAY4), - ICE_PTT(34, IP, IPV4, NOF, IP_IP, IPV4, NOF, SCTP, PAY4), - ICE_PTT(35, IP, IPV4, NOF, IP_IP, IPV4, NOF, ICMP, PAY4), - - /* IPv4 --> IPv6 */ - ICE_PTT(36, IP, IPV4, NOF, IP_IP, IPV6, FRG, NONE, PAY3), - ICE_PTT(37, IP, IPV4, NOF, IP_IP, IPV6, NOF, NONE, PAY3), - ICE_PTT(38, IP, IPV4, NOF, IP_IP, IPV6, NOF, UDP, PAY4), - ICE_PTT_UNUSED_ENTRY(39), - ICE_PTT(40, IP, IPV4, NOF, IP_IP, IPV6, NOF, TCP, PAY4), - ICE_PTT(41, IP, IPV4, NOF, IP_IP, IPV6, NOF, SCTP, PAY4), - ICE_PTT(42, IP, IPV4, NOF, IP_IP, IPV6, NOF, ICMP, PAY4), - - /* IPv4 --> GRE/NAT */ - ICE_PTT(43, IP, IPV4, NOF, IP_GRENAT, NONE, NOF, NONE, PAY3), - - /* IPv4 --> GRE/NAT --> IPv4 */ - ICE_PTT(44, IP, IPV4, NOF, IP_GRENAT, IPV4, FRG, NONE, PAY3), - ICE_PTT(45, IP, IPV4, NOF, IP_GRENAT, IPV4, NOF, NONE, PAY3), - ICE_PTT(46, IP, IPV4, NOF, IP_GRENAT, IPV4, NOF, UDP, PAY4), - ICE_PTT_UNUSED_ENTRY(47), - ICE_PTT(48, IP, IPV4, NOF, IP_GRENAT, IPV4, NOF, TCP, PAY4), - ICE_PTT(49, IP, IPV4, NOF, IP_GRENAT, IPV4, NOF, SCTP, PAY4), - ICE_PTT(50, IP, IPV4, NOF, IP_GRENAT, IPV4, NOF, ICMP, PAY4), - - /* IPv4 --> GRE/NAT --> IPv6 */ - ICE_PTT(51, IP, IPV4, NOF, IP_GRENAT, IPV6, FRG, NONE, PAY3), - ICE_PTT(52, IP, IPV4, NOF, IP_GRENAT, IPV6, NOF, NONE, PAY3), - ICE_PTT(53, IP, IPV4, NOF, IP_GRENAT, IPV6, NOF, UDP, PAY4), - ICE_PTT_UNUSED_ENTRY(54), - ICE_PTT(55, IP, IPV4, NOF, IP_GRENAT, IPV6, NOF, TCP, PAY4), - ICE_PTT(56, IP, IPV4, NOF, IP_GRENAT, IPV6, NOF, SCTP, PAY4), - ICE_PTT(57, IP, IPV4, NOF, IP_GRENAT, IPV6, NOF, ICMP, PAY4), - - /* IPv4 --> GRE/NAT --> MAC */ - ICE_PTT(58, IP, IPV4, NOF, IP_GRENAT_MAC, NONE, NOF, NONE, PAY3), - - /* IPv4 --> GRE/NAT --> MAC --> IPv4 */ - ICE_PTT(59, IP, IPV4, NOF, IP_GRENAT_MAC, IPV4, FRG, NONE, PAY3), - ICE_PTT(60, IP, IPV4, NOF, IP_GRENAT_MAC, IPV4, NOF, NONE, PAY3), - ICE_PTT(61, IP, IPV4, NOF, IP_GRENAT_MAC, IPV4, NOF, UDP, PAY4), - ICE_PTT_UNUSED_ENTRY(62), - ICE_PTT(63, IP, IPV4, NOF, IP_GRENAT_MAC, IPV4, NOF, TCP, PAY4), - ICE_PTT(64, IP, IPV4, NOF, IP_GRENAT_MAC, IPV4, NOF, SCTP, PAY4), - ICE_PTT(65, IP, IPV4, NOF, IP_GRENAT_MAC, IPV4, NOF, ICMP, PAY4), - - /* IPv4 --> GRE/NAT -> MAC --> IPv6 */ - ICE_PTT(66, IP, IPV4, NOF, IP_GRENAT_MAC, IPV6, FRG, NONE, PAY3), - ICE_PTT(67, IP, IPV4, NOF, IP_GRENAT_MAC, IPV6, NOF, NONE, PAY3), - ICE_PTT(68, IP, IPV4, NOF, IP_GRENAT_MAC, IPV6, NOF, UDP, PAY4), - ICE_PTT_UNUSED_ENTRY(69), - ICE_PTT(70, IP, IPV4, NOF, IP_GRENAT_MAC, IPV6, NOF, TCP, PAY4), - ICE_PTT(71, IP, IPV4, NOF, IP_GRENAT_MAC, IPV6, NOF, SCTP, PAY4), - ICE_PTT(72, IP, IPV4, NOF, IP_GRENAT_MAC, IPV6, NOF, ICMP, PAY4), - - /* IPv4 --> GRE/NAT --> MAC/VLAN */ - ICE_PTT(73, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, NONE, NOF, NONE, PAY3), - - /* IPv4 ---> GRE/NAT -> MAC/VLAN --> IPv4 */ - ICE_PTT(74, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, IPV4, FRG, NONE, PAY3), - ICE_PTT(75, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, IPV4, NOF, NONE, PAY3), - ICE_PTT(76, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, IPV4, NOF, UDP, PAY4), - ICE_PTT_UNUSED_ENTRY(77), - ICE_PTT(78, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, IPV4, NOF, TCP, PAY4), - ICE_PTT(79, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, IPV4, NOF, SCTP, PAY4), - ICE_PTT(80, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, IPV4, NOF, ICMP, PAY4), - - /* IPv4 -> GRE/NAT -> MAC/VLAN --> IPv6 */ - ICE_PTT(81, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, IPV6, FRG, NONE, PAY3), - ICE_PTT(82, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, IPV6, NOF, NONE, PAY3), - ICE_PTT(83, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, IPV6, NOF, UDP, PAY4), - ICE_PTT_UNUSED_ENTRY(84), - ICE_PTT(85, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, IPV6, NOF, TCP, PAY4), - ICE_PTT(86, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, IPV6, NOF, SCTP, PAY4), - ICE_PTT(87, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, IPV6, NOF, ICMP, PAY4), - - /* Non Tunneled IPv6 */ - ICE_PTT(88, IP, IPV6, FRG, NONE, NONE, NOF, NONE, PAY3), - ICE_PTT(89, IP, IPV6, NOF, NONE, NONE, NOF, NONE, PAY3), - ICE_PTT(90, IP, IPV6, NOF, NONE, NONE, NOF, UDP, PAY4), - ICE_PTT_UNUSED_ENTRY(91), - ICE_PTT(92, IP, IPV6, NOF, NONE, NONE, NOF, TCP, PAY4), - ICE_PTT(93, IP, IPV6, NOF, NONE, NONE, NOF, SCTP, PAY4), - ICE_PTT(94, IP, IPV6, NOF, NONE, NONE, NOF, ICMP, PAY4), - - /* IPv6 --> IPv4 */ - ICE_PTT(95, IP, IPV6, NOF, IP_IP, IPV4, FRG, NONE, PAY3), - ICE_PTT(96, IP, IPV6, NOF, IP_IP, IPV4, NOF, NONE, PAY3), - ICE_PTT(97, IP, IPV6, NOF, IP_IP, IPV4, NOF, UDP, PAY4), - ICE_PTT_UNUSED_ENTRY(98), - ICE_PTT(99, IP, IPV6, NOF, IP_IP, IPV4, NOF, TCP, PAY4), - ICE_PTT(100, IP, IPV6, NOF, IP_IP, IPV4, NOF, SCTP, PAY4), - ICE_PTT(101, IP, IPV6, NOF, IP_IP, IPV4, NOF, ICMP, PAY4), - - /* IPv6 --> IPv6 */ - ICE_PTT(102, IP, IPV6, NOF, IP_IP, IPV6, FRG, NONE, PAY3), - ICE_PTT(103, IP, IPV6, NOF, IP_IP, IPV6, NOF, NONE, PAY3), - ICE_PTT(104, IP, IPV6, NOF, IP_IP, IPV6, NOF, UDP, PAY4), - ICE_PTT_UNUSED_ENTRY(105), - ICE_PTT(106, IP, IPV6, NOF, IP_IP, IPV6, NOF, TCP, PAY4), - ICE_PTT(107, IP, IPV6, NOF, IP_IP, IPV6, NOF, SCTP, PAY4), - ICE_PTT(108, IP, IPV6, NOF, IP_IP, IPV6, NOF, ICMP, PAY4), - - /* IPv6 --> GRE/NAT */ - ICE_PTT(109, IP, IPV6, NOF, IP_GRENAT, NONE, NOF, NONE, PAY3), - - /* IPv6 --> GRE/NAT -> IPv4 */ - ICE_PTT(110, IP, IPV6, NOF, IP_GRENAT, IPV4, FRG, NONE, PAY3), - ICE_PTT(111, IP, IPV6, NOF, IP_GRENAT, IPV4, NOF, NONE, PAY3), - ICE_PTT(112, IP, IPV6, NOF, IP_GRENAT, IPV4, NOF, UDP, PAY4), - ICE_PTT_UNUSED_ENTRY(113), - ICE_PTT(114, IP, IPV6, NOF, IP_GRENAT, IPV4, NOF, TCP, PAY4), - ICE_PTT(115, IP, IPV6, NOF, IP_GRENAT, IPV4, NOF, SCTP, PAY4), - ICE_PTT(116, IP, IPV6, NOF, IP_GRENAT, IPV4, NOF, ICMP, PAY4), - - /* IPv6 --> GRE/NAT -> IPv6 */ - ICE_PTT(117, IP, IPV6, NOF, IP_GRENAT, IPV6, FRG, NONE, PAY3), - ICE_PTT(118, IP, IPV6, NOF, IP_GRENAT, IPV6, NOF, NONE, PAY3), - ICE_PTT(119, IP, IPV6, NOF, IP_GRENAT, IPV6, NOF, UDP, PAY4), - ICE_PTT_UNUSED_ENTRY(120), - ICE_PTT(121, IP, IPV6, NOF, IP_GRENAT, IPV6, NOF, TCP, PAY4), - ICE_PTT(122, IP, IPV6, NOF, IP_GRENAT, IPV6, NOF, SCTP, PAY4), - ICE_PTT(123, IP, IPV6, NOF, IP_GRENAT, IPV6, NOF, ICMP, PAY4), - - /* IPv6 --> GRE/NAT -> MAC */ - ICE_PTT(124, IP, IPV6, NOF, IP_GRENAT_MAC, NONE, NOF, NONE, PAY3), - - /* IPv6 --> GRE/NAT -> MAC -> IPv4 */ - ICE_PTT(125, IP, IPV6, NOF, IP_GRENAT_MAC, IPV4, FRG, NONE, PAY3), - ICE_PTT(126, IP, IPV6, NOF, IP_GRENAT_MAC, IPV4, NOF, NONE, PAY3), - ICE_PTT(127, IP, IPV6, NOF, IP_GRENAT_MAC, IPV4, NOF, UDP, PAY4), - ICE_PTT_UNUSED_ENTRY(128), - ICE_PTT(129, IP, IPV6, NOF, IP_GRENAT_MAC, IPV4, NOF, TCP, PAY4), - ICE_PTT(130, IP, IPV6, NOF, IP_GRENAT_MAC, IPV4, NOF, SCTP, PAY4), - ICE_PTT(131, IP, IPV6, NOF, IP_GRENAT_MAC, IPV4, NOF, ICMP, PAY4), - - /* IPv6 --> GRE/NAT -> MAC -> IPv6 */ - ICE_PTT(132, IP, IPV6, NOF, IP_GRENAT_MAC, IPV6, FRG, NONE, PAY3), - ICE_PTT(133, IP, IPV6, NOF, IP_GRENAT_MAC, IPV6, NOF, NONE, PAY3), - ICE_PTT(134, IP, IPV6, NOF, IP_GRENAT_MAC, IPV6, NOF, UDP, PAY4), - ICE_PTT_UNUSED_ENTRY(135), - ICE_PTT(136, IP, IPV6, NOF, IP_GRENAT_MAC, IPV6, NOF, TCP, PAY4), - ICE_PTT(137, IP, IPV6, NOF, IP_GRENAT_MAC, IPV6, NOF, SCTP, PAY4), - ICE_PTT(138, IP, IPV6, NOF, IP_GRENAT_MAC, IPV6, NOF, ICMP, PAY4), - - /* IPv6 --> GRE/NAT -> MAC/VLAN */ - ICE_PTT(139, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, NONE, NOF, NONE, PAY3), - - /* IPv6 --> GRE/NAT -> MAC/VLAN --> IPv4 */ - ICE_PTT(140, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV4, FRG, NONE, PAY3), - ICE_PTT(141, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV4, NOF, NONE, PAY3), - ICE_PTT(142, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV4, NOF, UDP, PAY4), - ICE_PTT_UNUSED_ENTRY(143), - ICE_PTT(144, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV4, NOF, TCP, PAY4), - ICE_PTT(145, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV4, NOF, SCTP, PAY4), - ICE_PTT(146, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV4, NOF, ICMP, PAY4), - - /* IPv6 --> GRE/NAT -> MAC/VLAN --> IPv6 */ - ICE_PTT(147, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV6, FRG, NONE, PAY3), - ICE_PTT(148, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV6, NOF, NONE, PAY3), - ICE_PTT(149, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV6, NOF, UDP, PAY4), - ICE_PTT_UNUSED_ENTRY(150), - ICE_PTT(151, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV6, NOF, TCP, PAY4), - ICE_PTT(152, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV6, NOF, SCTP, PAY4), - ICE_PTT(153, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV6, NOF, ICMP, PAY4), + ICE_PTYPES /* unused entries */ - [154 ... 1023] = { 0, 0, 0, 0, 0, 0, 0, 0, 0 } + [ICE_NUM_DEFINED_PTYPES ... 1023] = { 0, 0, 0, 0, 0, 0, 0, 0, 0 } }; static inline struct ice_rx_ptype_decoded ice_decode_rx_desc_ptype(u16 ptype) diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c index d1a14bf7b65c40..b97d116650beb0 100644 --- a/drivers/net/ethernet/intel/ice/ice_main.c +++ b/drivers/net/ethernet/intel/ice/ice_main.c @@ -1252,6 +1252,32 @@ ice_handle_link_event(struct ice_pf *pf, struct ice_rq_event_info *event) return status; } +/** + * ice_get_fwlog_data - copy the FW log data from ARQ event + * @pf: PF that the FW log event is associated with + * @event: event structure containing FW log data + */ +static void +ice_get_fwlog_data(struct ice_pf *pf, struct ice_rq_event_info *event) +{ + struct ice_fwlog_data *fwlog; + struct ice_hw *hw = &pf->hw; + + fwlog = &hw->fwlog_ring.rings[hw->fwlog_ring.tail]; + + memset(fwlog->data, 0, PAGE_SIZE); + fwlog->data_size = le16_to_cpu(event->desc.datalen); + + memcpy(fwlog->data, event->msg_buf, fwlog->data_size); + ice_fwlog_ring_increment(&hw->fwlog_ring.tail, hw->fwlog_ring.size); + + if (ice_fwlog_ring_full(&hw->fwlog_ring)) { + /* the rings are full so bump the head to create room */ + ice_fwlog_ring_increment(&hw->fwlog_ring.head, + hw->fwlog_ring.size); + } +} + /** * ice_aq_prep_for_event - Prepare to wait for an AdminQ event from firmware * @pf: pointer to the PF private structure @@ -1533,8 +1559,8 @@ static int __ice_clean_ctrlq(struct ice_pf *pf, enum ice_ctl_q q_type) ice_vc_process_vf_msg(pf, &event, &data); break; - case ice_aqc_opc_fw_logging: - ice_output_fw_log(hw, &event.desc, event.msg_buf); + case ice_aqc_opc_fw_logs_event: + ice_get_fwlog_data(pf, &event); break; case ice_aqc_opc_lldp_set_mib_change: ice_dcb_process_lldp_set_mib_change(pf, &event); @@ -3400,6 +3426,7 @@ static void ice_set_ops(struct ice_vsi *vsi) netdev->netdev_ops = &ice_netdev_ops; netdev->udp_tunnel_nic_info = &pf->hw.udp_tunnel_nic; + netdev->xdp_metadata_ops = &ice_xdp_md_ops; ice_set_ethtool_ops(netdev); if (vsi->type != ICE_VSI_PF) @@ -4363,6 +4390,19 @@ static void ice_print_wake_reason(struct ice_pf *pf) dev_info(ice_pf_to_dev(pf), "Wake reason: %s", wake_str); } +/** + * ice_pf_fwlog_update_module - update 1 module + * @pf: pointer to the PF struct + * @log_level: log_level to use for the @module + * @module: module to update + */ +void ice_pf_fwlog_update_module(struct ice_pf *pf, int log_level, int module) +{ + struct ice_hw *hw = &pf->hw; + + hw->fwlog_cfg.module_entries[module].log_level = log_level; +} + /** * ice_register_netdev - register netdev * @vsi: pointer to the VSI struct @@ -5210,6 +5250,8 @@ static void ice_remove(struct pci_dev *pdev) msleep(100); } + ice_debugfs_exit(); + if (test_bit(ICE_FLAG_SRIOV_ENA, pf->flags)) { set_bit(ICE_VF_RESETS_DISABLED, pf->state); ice_free_vfs(pf); @@ -5681,6 +5723,8 @@ static int __init ice_module_init(void) goto err_dest_wq; } + ice_debugfs_init(); + status = pci_register_driver(&ice_driver); if (status) { pr_err("failed to register PCI driver, err %d\n", status); @@ -5691,6 +5735,7 @@ static int __init ice_module_init(void) err_dest_lag_wq: destroy_workqueue(ice_lag_wq); + ice_debugfs_exit(); err_dest_wq: destroy_workqueue(ice_wq); return status; @@ -6049,6 +6094,23 @@ ice_fix_features(struct net_device *netdev, netdev_features_t features) return features; } +/** + * ice_set_rx_rings_vlan_proto - update rings with new stripped VLAN proto + * @vsi: PF's VSI + * @vlan_ethertype: VLAN ethertype (802.1Q or 802.1ad) in network byte order + * + * Store current stripped VLAN proto in ring packet context, + * so it can be accessed more efficiently by packet processing code. + */ +static void +ice_set_rx_rings_vlan_proto(struct ice_vsi *vsi, __be16 vlan_ethertype) +{ + u16 i; + + ice_for_each_alloc_rxq(vsi, i) + vsi->rx_rings[i]->pkt_ctx.vlan_proto = vlan_ethertype; +} + /** * ice_set_vlan_offload_features - set VLAN offload features for the PF VSI * @vsi: PF's VSI @@ -6091,6 +6153,9 @@ ice_set_vlan_offload_features(struct ice_vsi *vsi, netdev_features_t features) if (strip_err || insert_err) return -EIO; + ice_set_rx_rings_vlan_proto(vsi, enable_stripping ? + htons(vlan_ethertype) : 0); + return 0; } diff --git a/drivers/net/ethernet/intel/ice/ice_ptp.c b/drivers/net/ethernet/intel/ice/ice_ptp.c index e9e59f4b5580b2..239cf8a2ee8011 100644 --- a/drivers/net/ethernet/intel/ice/ice_ptp.c +++ b/drivers/net/ethernet/intel/ice/ice_ptp.c @@ -2129,30 +2129,26 @@ int ice_ptp_set_ts_config(struct ice_pf *pf, struct ifreq *ifr) } /** - * ice_ptp_rx_hwtstamp - Check for an Rx timestamp - * @rx_ring: Ring to get the VSI info + * ice_ptp_get_rx_hwts - Get packet Rx timestamp in ns * @rx_desc: Receive descriptor - * @skb: Particular skb to send timestamp with + * @pkt_ctx: Packet context to get the cached time * * The driver receives a notification in the receive descriptor with timestamp. - * The timestamp is in ns, so we must convert the result first. */ -void -ice_ptp_rx_hwtstamp(struct ice_rx_ring *rx_ring, - union ice_32b_rx_flex_desc *rx_desc, struct sk_buff *skb) +u64 ice_ptp_get_rx_hwts(const union ice_32b_rx_flex_desc *rx_desc, + const struct ice_pkt_ctx *pkt_ctx) { - struct skb_shared_hwtstamps *hwtstamps; u64 ts_ns, cached_time; u32 ts_high; if (!(rx_desc->wb.time_stamp_low & ICE_PTP_TS_VALID)) - return; + return 0; - cached_time = READ_ONCE(rx_ring->cached_phctime); + cached_time = READ_ONCE(pkt_ctx->cached_phctime); /* Do not report a timestamp if we don't have a cached PHC time */ if (!cached_time) - return; + return 0; /* Use ice_ptp_extend_32b_ts directly, using the ring-specific cached * PHC value, rather than accessing the PF. This also allows us to @@ -2163,9 +2159,7 @@ ice_ptp_rx_hwtstamp(struct ice_rx_ring *rx_ring, ts_high = le32_to_cpu(rx_desc->wb.flex_ts.ts_high); ts_ns = ice_ptp_extend_32b_ts(cached_time, ts_high); - hwtstamps = skb_hwtstamps(skb); - memset(hwtstamps, 0, sizeof(*hwtstamps)); - hwtstamps->hwtstamp = ns_to_ktime(ts_ns); + return ts_ns; } /** diff --git a/drivers/net/ethernet/intel/ice/ice_ptp.h b/drivers/net/ethernet/intel/ice/ice_ptp.h index d7928106140970..032653a7a133cd 100644 --- a/drivers/net/ethernet/intel/ice/ice_ptp.h +++ b/drivers/net/ethernet/intel/ice/ice_ptp.h @@ -298,9 +298,8 @@ void ice_ptp_extts_event(struct ice_pf *pf); s8 ice_ptp_request_ts(struct ice_ptp_tx *tx, struct sk_buff *skb); enum ice_tx_tstamp_work ice_ptp_process_ts(struct ice_pf *pf); -void -ice_ptp_rx_hwtstamp(struct ice_rx_ring *rx_ring, - union ice_32b_rx_flex_desc *rx_desc, struct sk_buff *skb); +u64 ice_ptp_get_rx_hwts(const union ice_32b_rx_flex_desc *rx_desc, + const struct ice_pkt_ctx *pkt_ctx); void ice_ptp_reset(struct ice_pf *pf); void ice_ptp_prepare_for_reset(struct ice_pf *pf); void ice_ptp_init(struct ice_pf *pf); @@ -329,9 +328,14 @@ static inline bool ice_ptp_process_ts(struct ice_pf *pf) { return true; } -static inline void -ice_ptp_rx_hwtstamp(struct ice_rx_ring *rx_ring, - union ice_32b_rx_flex_desc *rx_desc, struct sk_buff *skb) { } + +static inline u64 +ice_ptp_get_rx_hwts(const union ice_32b_rx_flex_desc *rx_desc, + const struct ice_pkt_ctx *pkt_ctx) +{ + return 0; +} + static inline void ice_ptp_reset(struct ice_pf *pf) { } static inline void ice_ptp_prepare_for_reset(struct ice_pf *pf) { } static inline void ice_ptp_init(struct ice_pf *pf) { } diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.c b/drivers/net/ethernet/intel/ice/ice_txrx.c index 9e97ea86306867..59617f055e3530 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx.c +++ b/drivers/net/ethernet/intel/ice/ice_txrx.c @@ -557,13 +557,14 @@ ice_rx_frame_truesize(struct ice_rx_ring *rx_ring, const unsigned int size) * @xdp_prog: XDP program to run * @xdp_ring: ring to be used for XDP_TX action * @rx_buf: Rx buffer to store the XDP action + * @eop_desc: Last descriptor in packet to read metadata from * * Returns any of ICE_XDP_{PASS, CONSUMED, TX, REDIR} */ static void ice_run_xdp(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp, struct bpf_prog *xdp_prog, struct ice_tx_ring *xdp_ring, - struct ice_rx_buf *rx_buf) + struct ice_rx_buf *rx_buf, union ice_32b_rx_flex_desc *eop_desc) { unsigned int ret = ICE_XDP_PASS; u32 act; @@ -571,6 +572,8 @@ ice_run_xdp(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp, if (!xdp_prog) goto exit; + ice_xdp_meta_set_desc(xdp, eop_desc); + act = bpf_prog_run_xdp(xdp_prog, xdp); switch (act) { case XDP_PASS: @@ -1180,8 +1183,7 @@ int ice_clean_rx_irq(struct ice_rx_ring *rx_ring, int budget) struct sk_buff *skb; unsigned int size; u16 stat_err_bits; - u16 vlan_tag = 0; - u16 rx_ptype; + u16 vlan_tci; /* get the Rx desc from Rx ring based on 'next_to_clean' */ rx_desc = ICE_RX_DESC(rx_ring, ntc); @@ -1241,7 +1243,7 @@ int ice_clean_rx_irq(struct ice_rx_ring *rx_ring, int budget) if (ice_is_non_eop(rx_ring, rx_desc)) continue; - ice_run_xdp(rx_ring, xdp, xdp_prog, xdp_ring, rx_buf); + ice_run_xdp(rx_ring, xdp, xdp_prog, xdp_ring, rx_buf, rx_desc); if (rx_buf->act == ICE_XDP_PASS) goto construct_skb; total_rx_bytes += xdp_get_buff_len(xdp); @@ -1276,7 +1278,7 @@ int ice_clean_rx_irq(struct ice_rx_ring *rx_ring, int budget) continue; } - vlan_tag = ice_get_vlan_tag_from_rx_desc(rx_desc); + vlan_tci = ice_get_vlan_tci(rx_desc); /* pad the skb if needed, to make a valid ethernet frame */ if (eth_skb_pad(skb)) @@ -1286,14 +1288,11 @@ int ice_clean_rx_irq(struct ice_rx_ring *rx_ring, int budget) total_rx_bytes += skb->len; /* populate checksum, VLAN, and protocol */ - rx_ptype = le16_to_cpu(rx_desc->wb.ptype_flex_flags0) & - ICE_RX_FLEX_DESC_PTYPE_M; - - ice_process_skb_fields(rx_ring, rx_desc, skb, rx_ptype); + ice_process_skb_fields(rx_ring, rx_desc, skb); ice_trace(clean_rx_irq_indicate, rx_ring, rx_desc, skb); /* send completed skb up the stack */ - ice_receive_skb(rx_ring, skb, vlan_tag); + ice_receive_skb(rx_ring, skb, vlan_tci); /* update budget accounting */ total_rx_pkts++; diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.h b/drivers/net/ethernet/intel/ice/ice_txrx.h index daf7b9dbb1435a..b3379ff7367478 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx.h +++ b/drivers/net/ethernet/intel/ice/ice_txrx.h @@ -257,6 +257,20 @@ enum ice_rx_dtype { ICE_RX_DTYPE_SPLIT_ALWAYS = 2, }; +struct ice_pkt_ctx { + u64 cached_phctime; + __be16 vlan_proto; +}; + +struct ice_xdp_buff { + struct xdp_buff xdp_buff; + const union ice_32b_rx_flex_desc *eop_desc; + const struct ice_pkt_ctx *pkt_ctx; +}; + +/* Required for compatibility with xdp_buffs from xsk_pool */ +static_assert(offsetof(struct ice_xdp_buff, xdp_buff) == 0); + /* indices into GLINT_ITR registers */ #define ICE_RX_ITR ICE_IDX_ITR0 #define ICE_TX_ITR ICE_IDX_ITR1 @@ -298,7 +312,6 @@ enum ice_dynamic_itr { /* descriptor ring, associated with a VSI */ struct ice_rx_ring { /* CL1 - 1st cacheline starts here */ - struct ice_rx_ring *next; /* pointer to next ring in q_vector */ void *desc; /* Descriptor ring memory */ struct device *dev; /* Used for DMA mapping */ struct net_device *netdev; /* netdev ring maps to */ @@ -310,13 +323,24 @@ struct ice_rx_ring { u16 count; /* Number of descriptors */ u16 reg_idx; /* HW register index of the ring */ u16 next_to_alloc; - /* CL2 - 2nd cacheline starts here */ + union { struct ice_rx_buf *rx_buf; struct xdp_buff **xdp_buf; }; - struct xdp_buff xdp; + /* CL2 - 2nd cacheline starts here */ + union { + struct ice_xdp_buff xdp_ext; + struct xdp_buff xdp; + }; /* CL3 - 3rd cacheline starts here */ + union { + struct ice_pkt_ctx pkt_ctx; + struct { + u64 cached_phctime; + __be16 vlan_proto; + }; + }; struct bpf_prog *xdp_prog; u16 rx_offset; @@ -332,9 +356,9 @@ struct ice_rx_ring { /* CL4 - 4th cacheline starts here */ struct ice_channel *ch; struct ice_tx_ring *xdp_ring; + struct ice_rx_ring *next; /* pointer to next ring in q_vector */ struct xsk_buff_pool *xsk_pool; dma_addr_t dma; /* physical address of ring */ - u64 cached_phctime; u16 rx_buf_len; u8 dcb_tc; /* Traffic class of ring */ u8 ptp_rx; diff --git a/drivers/net/ethernet/intel/ice/ice_txrx_lib.c b/drivers/net/ethernet/intel/ice/ice_txrx_lib.c index 7e06373e14d98e..839e5da24ad583 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_txrx_lib.c @@ -63,28 +63,42 @@ static enum pkt_hash_types ice_ptype_to_htype(u16 ptype) } /** - * ice_rx_hash - set the hash value in the skb + * ice_get_rx_hash - get RX hash value from descriptor + * @rx_desc: specific descriptor + * + * Returns hash, if present, 0 otherwise. + */ +static u32 ice_get_rx_hash(const union ice_32b_rx_flex_desc *rx_desc) +{ + const struct ice_32b_rx_flex_desc_nic *nic_mdid; + + if (unlikely(rx_desc->wb.rxdid != ICE_RXDID_FLEX_NIC)) + return 0; + + nic_mdid = (struct ice_32b_rx_flex_desc_nic *)rx_desc; + return le32_to_cpu(nic_mdid->rss_hash); +} + +/** + * ice_rx_hash_to_skb - set the hash value in the skb * @rx_ring: descriptor ring * @rx_desc: specific descriptor * @skb: pointer to current skb * @rx_ptype: the ptype value from the descriptor */ static void -ice_rx_hash(struct ice_rx_ring *rx_ring, union ice_32b_rx_flex_desc *rx_desc, - struct sk_buff *skb, u16 rx_ptype) +ice_rx_hash_to_skb(const struct ice_rx_ring *rx_ring, + const union ice_32b_rx_flex_desc *rx_desc, + struct sk_buff *skb, u16 rx_ptype) { - struct ice_32b_rx_flex_desc_nic *nic_mdid; u32 hash; if (!(rx_ring->netdev->features & NETIF_F_RXHASH)) return; - if (rx_desc->wb.rxdid != ICE_RXDID_FLEX_NIC) - return; - - nic_mdid = (struct ice_32b_rx_flex_desc_nic *)rx_desc; - hash = le32_to_cpu(nic_mdid->rss_hash); - skb_set_hash(skb, hash, ice_ptype_to_htype(rx_ptype)); + hash = ice_get_rx_hash(rx_desc); + if (likely(hash)) + skb_set_hash(skb, hash, ice_ptype_to_htype(rx_ptype)); } /** @@ -170,12 +184,39 @@ ice_rx_csum(struct ice_rx_ring *ring, struct sk_buff *skb, ring->vsi->back->hw_csum_rx_error++; } +/** + * ice_ptp_rx_hwts_to_skb - Put RX timestamp into skb + * @rx_ring: Ring to get the VSI info + * @rx_desc: Receive descriptor + * @skb: Particular skb to send timestamp with + * + * The timestamp is in ns, so we must convert the result first. + */ +static void +ice_ptp_rx_hwts_to_skb(struct ice_rx_ring *rx_ring, + const union ice_32b_rx_flex_desc *rx_desc, + struct sk_buff *skb) +{ + u64 ts_ns = ice_ptp_get_rx_hwts(rx_desc, &rx_ring->pkt_ctx); + + skb_hwtstamps(skb)->hwtstamp = ns_to_ktime(ts_ns); +} + +/** + * ice_get_ptype - Read HW packet type from the descriptor + * @rx_desc: RX descriptor + */ +static u16 ice_get_ptype(const union ice_32b_rx_flex_desc *rx_desc) +{ + return le16_to_cpu(rx_desc->wb.ptype_flex_flags0) & + ICE_RX_FLEX_DESC_PTYPE_M; +} + /** * ice_process_skb_fields - Populate skb header fields from Rx descriptor * @rx_ring: Rx descriptor ring packet is being transacted on * @rx_desc: pointer to the EOP Rx descriptor * @skb: pointer to current skb being populated - * @ptype: the packet type decoded by hardware * * This function checks the ring, descriptor, and packet information in * order to populate the hash, checksum, VLAN, protocol, and @@ -184,9 +225,11 @@ ice_rx_csum(struct ice_rx_ring *ring, struct sk_buff *skb, void ice_process_skb_fields(struct ice_rx_ring *rx_ring, union ice_32b_rx_flex_desc *rx_desc, - struct sk_buff *skb, u16 ptype) + struct sk_buff *skb) { - ice_rx_hash(rx_ring, rx_desc, skb, ptype); + u16 ptype = ice_get_ptype(rx_desc); + + ice_rx_hash_to_skb(rx_ring, rx_desc, skb, ptype); /* modifies the skb - consumes the enet header */ skb->protocol = eth_type_trans(skb, rx_ring->netdev); @@ -194,28 +237,24 @@ ice_process_skb_fields(struct ice_rx_ring *rx_ring, ice_rx_csum(rx_ring, skb, rx_desc, ptype); if (rx_ring->ptp_rx) - ice_ptp_rx_hwtstamp(rx_ring, rx_desc, skb); + ice_ptp_rx_hwts_to_skb(rx_ring, rx_desc, skb); } /** * ice_receive_skb - Send a completed packet up the stack * @rx_ring: Rx ring in play * @skb: packet to send up - * @vlan_tag: VLAN tag for packet + * @vlan_tci: VLAN TCI for packet * * This function sends the completed packet (via. skb) up the stack using * gro receive functions (with/without VLAN tag) */ void -ice_receive_skb(struct ice_rx_ring *rx_ring, struct sk_buff *skb, u16 vlan_tag) +ice_receive_skb(struct ice_rx_ring *rx_ring, struct sk_buff *skb, u16 vlan_tci) { - netdev_features_t features = rx_ring->netdev->features; - bool non_zero_vlan = !!(vlan_tag & VLAN_VID_MASK); - - if ((features & NETIF_F_HW_VLAN_CTAG_RX) && non_zero_vlan) - __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tag); - else if ((features & NETIF_F_HW_VLAN_STAG_RX) && non_zero_vlan) - __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021AD), vlan_tag); + if ((vlan_tci & VLAN_VID_MASK) && rx_ring->vlan_proto) + __vlan_hwaccel_put_tag(skb, rx_ring->vlan_proto, + vlan_tci); napi_gro_receive(&rx_ring->q_vector->napi, skb); } @@ -464,3 +503,125 @@ void ice_finalize_xdp_rx(struct ice_tx_ring *xdp_ring, unsigned int xdp_res, spin_unlock(&xdp_ring->tx_lock); } } + +/** + * ice_xdp_rx_hw_ts - HW timestamp XDP hint handler + * @ctx: XDP buff pointer + * @ts_ns: destination address + * + * Copy HW timestamp (if available) to the destination address. + */ +static int ice_xdp_rx_hw_ts(const struct xdp_md *ctx, u64 *ts_ns) +{ + const struct ice_xdp_buff *xdp_ext = (void *)ctx; + + *ts_ns = ice_ptp_get_rx_hwts(xdp_ext->eop_desc, + xdp_ext->pkt_ctx); + if (!*ts_ns) + return -ENODATA; + + return 0; +} + +/* Define a ptype index -> XDP hash type lookup table. + * It uses the same ptype definitions as ice_decode_rx_desc_ptype[], + * avoiding possible copy-paste errors. + */ +#undef ICE_PTT +#undef ICE_PTT_UNUSED_ENTRY + +#define ICE_PTT(PTYPE, OUTER_IP, OUTER_IP_VER, OUTER_FRAG, T, TE, TEF, I, PL)\ + [PTYPE] = XDP_RSS_L3_##OUTER_IP_VER | XDP_RSS_L4_##I | XDP_RSS_TYPE_##PL + +#define ICE_PTT_UNUSED_ENTRY(PTYPE) [PTYPE] = 0 + +/* A few supplementary definitions for when XDP hash types do not coincide + * with what can be generated from ptype definitions + * by means of preprocessor concatenation. + */ +#define XDP_RSS_L3_NONE XDP_RSS_TYPE_NONE +#define XDP_RSS_L4_NONE XDP_RSS_TYPE_NONE +#define XDP_RSS_TYPE_PAY2 XDP_RSS_TYPE_L2 +#define XDP_RSS_TYPE_PAY3 XDP_RSS_TYPE_NONE +#define XDP_RSS_TYPE_PAY4 XDP_RSS_L4 + +static const enum xdp_rss_hash_type +ice_ptype_to_xdp_hash[ICE_NUM_DEFINED_PTYPES] = { + ICE_PTYPES +}; + +#undef XDP_RSS_L3_NONE +#undef XDP_RSS_L4_NONE +#undef XDP_RSS_TYPE_PAY2 +#undef XDP_RSS_TYPE_PAY3 +#undef XDP_RSS_TYPE_PAY4 + +#undef ICE_PTT +#undef ICE_PTT_UNUSED_ENTRY + +/** + * ice_xdp_rx_hash_type - Get XDP-specific hash type from the RX descriptor + * @eop_desc: End of Packet descriptor + */ +static enum xdp_rss_hash_type +ice_xdp_rx_hash_type(const union ice_32b_rx_flex_desc *eop_desc) +{ + u16 ptype = ice_get_ptype(eop_desc); + + if (unlikely(ptype >= ICE_NUM_DEFINED_PTYPES)) + return 0; + + return ice_ptype_to_xdp_hash[ptype]; +} + +/** + * ice_xdp_rx_hash - RX hash XDP hint handler + * @ctx: XDP buff pointer + * @hash: hash destination address + * @rss_type: XDP hash type destination address + * + * Copy RX hash (if available) and its type to the destination address. + */ +static int ice_xdp_rx_hash(const struct xdp_md *ctx, u32 *hash, + enum xdp_rss_hash_type *rss_type) +{ + const struct ice_xdp_buff *xdp_ext = (void *)ctx; + + *hash = ice_get_rx_hash(xdp_ext->eop_desc); + *rss_type = ice_xdp_rx_hash_type(xdp_ext->eop_desc); + if (!likely(*hash)) + return -ENODATA; + + return 0; +} + +/** + * ice_xdp_rx_vlan_tag - VLAN tag XDP hint handler + * @ctx: XDP buff pointer + * @vlan_proto: destination address for VLAN protocol + * @vlan_tci: destination address for VLAN TCI + * + * Copy VLAN tag (if was stripped) and corresponding protocol + * to the destination address. + */ +static int ice_xdp_rx_vlan_tag(const struct xdp_md *ctx, __be16 *vlan_proto, + u16 *vlan_tci) +{ + const struct ice_xdp_buff *xdp_ext = (void *)ctx; + + *vlan_proto = xdp_ext->pkt_ctx->vlan_proto; + if (!*vlan_proto) + return -ENODATA; + + *vlan_tci = ice_get_vlan_tci(xdp_ext->eop_desc); + if (!*vlan_tci) + return -ENODATA; + + return 0; +} + +const struct xdp_metadata_ops ice_xdp_md_ops = { + .xmo_rx_timestamp = ice_xdp_rx_hw_ts, + .xmo_rx_hash = ice_xdp_rx_hash, + .xmo_rx_vlan_tag = ice_xdp_rx_vlan_tag, +}; diff --git a/drivers/net/ethernet/intel/ice/ice_txrx_lib.h b/drivers/net/ethernet/intel/ice/ice_txrx_lib.h index 115969ecdf7b97..76204750861960 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx_lib.h +++ b/drivers/net/ethernet/intel/ice/ice_txrx_lib.h @@ -84,7 +84,7 @@ ice_build_ctob(u64 td_cmd, u64 td_offset, unsigned int size, u64 td_tag) } /** - * ice_get_vlan_tag_from_rx_desc - get VLAN from Rx flex descriptor + * ice_get_vlan_tci - get VLAN TCI from Rx flex descriptor * @rx_desc: Rx 32b flex descriptor with RXDID=2 * * The OS and current PF implementation only support stripping a single VLAN tag @@ -92,7 +92,7 @@ ice_build_ctob(u64 td_cmd, u64 td_offset, unsigned int size, u64 td_tag) * one is found return the tag, else return 0 to mean no VLAN tag was found. */ static inline u16 -ice_get_vlan_tag_from_rx_desc(union ice_32b_rx_flex_desc *rx_desc) +ice_get_vlan_tci(const union ice_32b_rx_flex_desc *rx_desc) { u16 stat_err_bits; @@ -148,7 +148,17 @@ void ice_release_rx_desc(struct ice_rx_ring *rx_ring, u16 val); void ice_process_skb_fields(struct ice_rx_ring *rx_ring, union ice_32b_rx_flex_desc *rx_desc, - struct sk_buff *skb, u16 ptype); + struct sk_buff *skb); void -ice_receive_skb(struct ice_rx_ring *rx_ring, struct sk_buff *skb, u16 vlan_tag); +ice_receive_skb(struct ice_rx_ring *rx_ring, struct sk_buff *skb, u16 vlan_tci); + +static inline void +ice_xdp_meta_set_desc(struct xdp_buff *xdp, + union ice_32b_rx_flex_desc *eop_desc) +{ + struct ice_xdp_buff *xdp_ext = container_of(xdp, struct ice_xdp_buff, + xdp_buff); + + xdp_ext->eop_desc = eop_desc; +} #endif /* !_ICE_TXRX_LIB_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_type.h b/drivers/net/ethernet/intel/ice/ice_type.h index 8489248db0fc99..6df7c4487ad0fc 100644 --- a/drivers/net/ethernet/intel/ice/ice_type.h +++ b/drivers/net/ethernet/intel/ice/ice_type.h @@ -17,6 +17,7 @@ #include "ice_protocol_type.h" #include "ice_sbq_cmd.h" #include "ice_vlan_mode.h" +#include "ice_fwlog.h" static inline bool ice_is_tc_ena(unsigned long bitmap, u8 tc) { @@ -738,24 +739,6 @@ struct ice_switch_info { DECLARE_BITMAP(prof_res_bm[ICE_MAX_NUM_PROFILES], ICE_MAX_FV_WORDS); }; -/* FW logging configuration */ -struct ice_fw_log_evnt { - u8 cfg : 4; /* New event enables to configure */ - u8 cur : 4; /* Current/active event enables */ -}; - -struct ice_fw_log_cfg { - u8 cq_en : 1; /* FW logging is enabled via the control queue */ - u8 uart_en : 1; /* FW logging is enabled via UART for all PFs */ - u8 actv_evnts; /* Cumulation of currently enabled log events */ - -#define ICE_FW_LOG_EVNT_INFO (ICE_AQC_FW_LOG_INFO_EN >> ICE_AQC_FW_LOG_EN_S) -#define ICE_FW_LOG_EVNT_INIT (ICE_AQC_FW_LOG_INIT_EN >> ICE_AQC_FW_LOG_EN_S) -#define ICE_FW_LOG_EVNT_FLOW (ICE_AQC_FW_LOG_FLOW_EN >> ICE_AQC_FW_LOG_EN_S) -#define ICE_FW_LOG_EVNT_ERR (ICE_AQC_FW_LOG_ERR_EN >> ICE_AQC_FW_LOG_EN_S) - struct ice_fw_log_evnt evnts[ICE_AQC_FW_LOG_ID_MAX]; -}; - /* Enum defining the different states of the mailbox snapshot in the * PF-VF mailbox overflow detection algorithm. The snapshot can be in * states: @@ -897,7 +880,9 @@ struct ice_hw { u8 fw_patch; /* firmware patch version */ u32 fw_build; /* firmware build number */ - struct ice_fw_log_cfg fw_log; + struct ice_fwlog_cfg fwlog_cfg; + bool fwlog_supported; /* does hardware support FW logging? */ + struct ice_fwlog_ring fwlog_ring; /* Device max aggregate bandwidths corresponding to the GL_PWR_MODE_CTL * register. Used for determining the ITR/INTRL granularity during diff --git a/drivers/net/ethernet/intel/ice/ice_xsk.c b/drivers/net/ethernet/intel/ice/ice_xsk.c index 99954508184f9d..5d1ae8e4058a4a 100644 --- a/drivers/net/ethernet/intel/ice/ice_xsk.c +++ b/drivers/net/ethernet/intel/ice/ice_xsk.c @@ -458,6 +458,11 @@ static u16 ice_fill_rx_descs(struct xsk_buff_pool *pool, struct xdp_buff **xdp, rx_desc->read.pkt_addr = cpu_to_le64(dma); rx_desc->wb.status_error0 = 0; + /* Put private info that changes on a per-packet basis + * into xdp_buff_xsk->cb. + */ + ice_xdp_meta_set_desc(*xdp, rx_desc); + rx_desc++; xdp++; } @@ -863,8 +868,7 @@ int ice_clean_rx_irq_zc(struct ice_rx_ring *rx_ring, int budget) struct xdp_buff *xdp; struct sk_buff *skb; u16 stat_err_bits; - u16 vlan_tag = 0; - u16 rx_ptype; + u16 vlan_tci; rx_desc = ICE_RX_DESC(rx_ring, ntc); @@ -942,13 +946,10 @@ int ice_clean_rx_irq_zc(struct ice_rx_ring *rx_ring, int budget) total_rx_bytes += skb->len; total_rx_packets++; - vlan_tag = ice_get_vlan_tag_from_rx_desc(rx_desc); - - rx_ptype = le16_to_cpu(rx_desc->wb.ptype_flex_flags0) & - ICE_RX_FLEX_DESC_PTYPE_M; + vlan_tci = ice_get_vlan_tci(rx_desc); - ice_process_skb_fields(rx_ring, rx_desc, skb, rx_ptype); - ice_receive_skb(rx_ring, skb, vlan_tag); + ice_process_skb_fields(rx_ring, rx_desc, skb); + ice_receive_skb(rx_ring, skb, vlan_tci); } rx_ring->next_to_clean = ntc; diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h index 1f3ff4bb875309..6446dc38998997 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h @@ -949,6 +949,8 @@ int npc_install_mcam_drop_rule(struct rvu *rvu, int mcam_idx, u16 *counter_idx, u64 bcast_mcast_val, u64 bcast_mcast_mask); void npc_mcam_rsrcs_reserve(struct rvu *rvu, int blkaddr, int entry_idx); bool npc_is_feature_supported(struct rvu *rvu, u64 features, u8 intf); +int npc_mcam_rsrcs_init(struct rvu *rvu, int blkaddr); +void npc_mcam_rsrcs_deinit(struct rvu *rvu); /* CPT APIs */ int rvu_cpt_register_interrupts(struct rvu *rvu); diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_devlink.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_devlink.c index 21b5d71c1e3758..bb5fdb225dab46 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_devlink.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_devlink.c @@ -1237,6 +1237,7 @@ enum rvu_af_dl_param_id { RVU_AF_DEVLINK_PARAM_ID_DWRR_MTU, RVU_AF_DEVLINK_PARAM_ID_NPC_EXACT_FEATURE_DISABLE, RVU_AF_DEVLINK_PARAM_ID_NPC_MCAM_ZONE_PERCENT, + RVU_AF_DEVLINK_PARAM_ID_NIX_MAXLF, }; static int rvu_af_npc_exact_feature_get(struct devlink *devlink, u32 id, @@ -1354,6 +1355,79 @@ static int rvu_af_dl_npc_mcam_high_zone_percent_validate(struct devlink *devlink return 0; } +static int rvu_af_dl_nix_maxlf_get(struct devlink *devlink, u32 id, + struct devlink_param_gset_ctx *ctx) +{ + struct rvu_devlink *rvu_dl = devlink_priv(devlink); + struct rvu *rvu = rvu_dl->rvu; + + ctx->val.vu16 = (u16)rvu_get_nixlf_count(rvu); + + return 0; +} + +static int rvu_af_dl_nix_maxlf_set(struct devlink *devlink, u32 id, + struct devlink_param_gset_ctx *ctx) +{ + struct rvu_devlink *rvu_dl = devlink_priv(devlink); + struct rvu *rvu = rvu_dl->rvu; + struct rvu_block *block; + int blkaddr = 0; + + npc_mcam_rsrcs_deinit(rvu); + blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); + while (blkaddr) { + block = &rvu->hw->block[blkaddr]; + block->lf.max = ctx->val.vu16; + blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); + } + + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + npc_mcam_rsrcs_init(rvu, blkaddr); + + return 0; +} + +static int rvu_af_dl_nix_maxlf_validate(struct devlink *devlink, u32 id, + union devlink_param_value val, + struct netlink_ext_ack *extack) +{ + struct rvu_devlink *rvu_dl = devlink_priv(devlink); + struct rvu *rvu = rvu_dl->rvu; + u16 max_nix0_lf, max_nix1_lf; + struct npc_mcam *mcam; + u64 cfg; + + cfg = rvu_read64(rvu, BLKADDR_NIX0, NIX_AF_CONST2); + max_nix0_lf = cfg & 0xFFF; + cfg = rvu_read64(rvu, BLKADDR_NIX1, NIX_AF_CONST2); + max_nix1_lf = cfg & 0xFFF; + + /* Do not allow user to modify maximum NIX LFs while mcam entries + * have already been assigned. + */ + mcam = &rvu->hw->mcam; + if (mcam->bmap_fcnt < mcam->bmap_entries) { + NL_SET_ERR_MSG_MOD(extack, + "mcam entries have already been assigned, can't resize"); + return -EPERM; + } + + if (max_nix0_lf && val.vu16 > max_nix0_lf) { + NL_SET_ERR_MSG_MOD(extack, + "requested nixlf is greater than the max supported nix0_lf"); + return -EPERM; + } + + if (max_nix1_lf && val.vu16 > max_nix1_lf) { + NL_SET_ERR_MSG_MOD(extack, + "requested nixlf is greater than the max supported nix1_lf"); + return -EINVAL; + } + + return 0; +} + static const struct devlink_param rvu_af_dl_params[] = { DEVLINK_PARAM_DRIVER(RVU_AF_DEVLINK_PARAM_ID_DWRR_MTU, "dwrr_mtu", DEVLINK_PARAM_TYPE_U32, @@ -1375,6 +1449,12 @@ static const struct devlink_param rvu_af_dl_param_exact_match[] = { rvu_af_dl_npc_mcam_high_zone_percent_get, rvu_af_dl_npc_mcam_high_zone_percent_set, rvu_af_dl_npc_mcam_high_zone_percent_validate), + DEVLINK_PARAM_DRIVER(RVU_AF_DEVLINK_PARAM_ID_NIX_MAXLF, + "nix_maxlf", DEVLINK_PARAM_TYPE_U16, + BIT(DEVLINK_PARAM_CMODE_RUNTIME), + rvu_af_dl_nix_maxlf_get, + rvu_af_dl_nix_maxlf_set, + rvu_af_dl_nix_maxlf_validate), }; /* Devlink switch mode */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c index b01503acd5208a..72e0a7717c3ebf 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c @@ -6142,14 +6142,12 @@ static struct nix_mcast_grp_elem *rvu_nix_mcast_find_grp_elem(struct nix_mcast_g struct nix_mcast_grp_elem *iter; bool is_found = false; - mutex_lock(&mcast_grp->mcast_grp_lock); list_for_each_entry(iter, &mcast_grp->mcast_grp_head, list) { if (iter->mcast_grp_idx == mcast_grp_idx) { is_found = true; break; } } - mutex_unlock(&mcast_grp->mcast_grp_lock); if (is_found) return iter; @@ -6162,7 +6160,7 @@ int rvu_nix_mcast_get_mce_index(struct rvu *rvu, u16 pcifunc, u32 mcast_grp_idx) struct nix_mcast_grp_elem *elem; struct nix_mcast_grp *mcast_grp; struct nix_hw *nix_hw; - int blkaddr; + int blkaddr, ret; blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); nix_hw = get_nix_hw(rvu->hw, blkaddr); @@ -6170,11 +6168,15 @@ int rvu_nix_mcast_get_mce_index(struct rvu *rvu, u16 pcifunc, u32 mcast_grp_idx) return NIX_AF_ERR_INVALID_NIXBLK; mcast_grp = &nix_hw->mcast_grp; + mutex_lock(&mcast_grp->mcast_grp_lock); elem = rvu_nix_mcast_find_grp_elem(mcast_grp, mcast_grp_idx); if (!elem) - return NIX_AF_ERR_INVALID_MCAST_GRP; + ret = NIX_AF_ERR_INVALID_MCAST_GRP; + else + ret = elem->mce_start_index; - return elem->mce_start_index; + mutex_unlock(&mcast_grp->mcast_grp_lock); + return ret; } void rvu_nix_mcast_flr_free_entries(struct rvu *rvu, u16 pcifunc) @@ -6238,7 +6240,7 @@ int rvu_nix_mcast_update_mcam_entry(struct rvu *rvu, u16 pcifunc, struct nix_mcast_grp_elem *elem; struct nix_mcast_grp *mcast_grp; struct nix_hw *nix_hw; - int blkaddr; + int blkaddr, ret = 0; blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); nix_hw = get_nix_hw(rvu->hw, blkaddr); @@ -6246,13 +6248,15 @@ int rvu_nix_mcast_update_mcam_entry(struct rvu *rvu, u16 pcifunc, return NIX_AF_ERR_INVALID_NIXBLK; mcast_grp = &nix_hw->mcast_grp; + mutex_lock(&mcast_grp->mcast_grp_lock); elem = rvu_nix_mcast_find_grp_elem(mcast_grp, mcast_grp_idx); if (!elem) - return NIX_AF_ERR_INVALID_MCAST_GRP; - - elem->mcam_index = mcam_index; + ret = NIX_AF_ERR_INVALID_MCAST_GRP; + else + elem->mcam_index = mcam_index; - return 0; + mutex_unlock(&mcast_grp->mcast_grp_lock); + return ret; } int rvu_mbox_handler_nix_mcast_grp_create(struct rvu *rvu, @@ -6297,18 +6301,27 @@ int rvu_mbox_handler_nix_mcast_grp_destroy(struct rvu *rvu, struct npc_delete_flow_rsp uninstall_rsp = { 0 }; struct nix_mcast_grp_elem *elem; struct nix_mcast_grp *mcast_grp; + int blkaddr, err, ret = 0; struct nix_mcast *mcast; struct nix_hw *nix_hw; - int blkaddr, err; err = nix_get_struct_ptrs(rvu, req->hdr.pcifunc, &nix_hw, &blkaddr); if (err) return err; mcast_grp = &nix_hw->mcast_grp; + + /* If AF is requesting for the deletion, + * then AF is already taking the lock + */ + if (!req->is_af) + mutex_lock(&mcast_grp->mcast_grp_lock); + elem = rvu_nix_mcast_find_grp_elem(mcast_grp, req->mcast_grp_idx); - if (!elem) - return NIX_AF_ERR_INVALID_MCAST_GRP; + if (!elem) { + ret = NIX_AF_ERR_INVALID_MCAST_GRP; + goto unlock_grp; + } /* If no mce entries are associated with the group * then just remove it from the global list. @@ -6333,19 +6346,15 @@ int rvu_mbox_handler_nix_mcast_grp_destroy(struct rvu *rvu, mutex_unlock(&mcast->mce_lock); delete_grp: - /* If AF is requesting for the deletion, - * then AF is already taking the lock - */ - if (!req->is_af) - mutex_lock(&mcast_grp->mcast_grp_lock); - list_del(&elem->list); kfree(elem); mcast_grp->count--; + +unlock_grp: if (!req->is_af) mutex_unlock(&mcast_grp->mcast_grp_lock); - return 0; + return ret; } int rvu_mbox_handler_nix_mcast_grp_update(struct rvu *rvu, @@ -6370,9 +6379,18 @@ int rvu_mbox_handler_nix_mcast_grp_update(struct rvu *rvu, return err; mcast_grp = &nix_hw->mcast_grp; + + /* If AF is requesting for the updation, + * then AF is already taking the lock + */ + if (!req->is_af) + mutex_lock(&mcast_grp->mcast_grp_lock); + elem = rvu_nix_mcast_find_grp_elem(mcast_grp, req->mcast_grp_idx); - if (!elem) - return NIX_AF_ERR_INVALID_MCAST_GRP; + if (!elem) { + ret = NIX_AF_ERR_INVALID_MCAST_GRP; + goto unlock_grp; + } /* If any pcifunc matches the group's pcifunc, then we can * delete the entire group. @@ -6383,9 +6401,10 @@ int rvu_mbox_handler_nix_mcast_grp_update(struct rvu *rvu, /* Delete group */ dreq.hdr.pcifunc = elem->pcifunc; dreq.mcast_grp_idx = elem->mcast_grp_idx; - dreq.is_af = req->is_af; + dreq.is_af = 1; rvu_mbox_handler_nix_mcast_grp_destroy(rvu, &dreq, NULL); - return 0; + ret = 0; + goto unlock_grp; } } } @@ -6410,7 +6429,7 @@ int rvu_mbox_handler_nix_mcast_grp_update(struct rvu *rvu, npc_enable_mcam_entry(rvu, mcam, npc_blkaddr, elem->mcam_index, true); ret = NIX_AF_ERR_NON_CONTIG_MCE_LIST; - goto done; + goto unlock_mce; } } @@ -6426,7 +6445,7 @@ int rvu_mbox_handler_nix_mcast_grp_update(struct rvu *rvu, npc_enable_mcam_entry(rvu, mcam, npc_blkaddr, elem->mcam_index, true); - goto done; + goto unlock_mce; } } else { if (!prev_count || prev_count < req->num_mce_entry) { @@ -6434,7 +6453,7 @@ int rvu_mbox_handler_nix_mcast_grp_update(struct rvu *rvu, npc_enable_mcam_entry(rvu, mcam, npc_blkaddr, elem->mcam_index, true); ret = NIX_AF_ERR_INVALID_MCAST_DEL_REQ; - goto done; + goto unlock_mce; } nix_free_mce_list(mcast, prev_count, elem->mce_start_index, elem->dir); @@ -6450,14 +6469,14 @@ int rvu_mbox_handler_nix_mcast_grp_update(struct rvu *rvu, elem->mcam_index, true); - goto done; + goto unlock_mce; } } if (elem->mcam_index == -1) { rsp->mce_start_index = elem->mce_start_index; ret = 0; - goto done; + goto unlock_mce; } nix_mcast_update_action(rvu, elem); @@ -6465,7 +6484,12 @@ int rvu_mbox_handler_nix_mcast_grp_update(struct rvu *rvu, rsp->mce_start_index = elem->mce_start_index; ret = 0; -done: +unlock_mce: mutex_unlock(&mcast->mce_lock); + +unlock_grp: + if (!req->is_af) + mutex_unlock(&mcast_grp->mcast_grp_lock); + return ret; } diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c index 2897a8b8e558ca..513c4fe8696750 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c @@ -1846,7 +1846,21 @@ static void npc_parser_profile_init(struct rvu *rvu, int blkaddr) npc_program_kpu_profile(rvu, blkaddr, idx, &rvu->kpu.kpu[idx]); } -static int npc_mcam_rsrcs_init(struct rvu *rvu, int blkaddr) +void npc_mcam_rsrcs_deinit(struct rvu *rvu) +{ + struct npc_mcam *mcam = &rvu->hw->mcam; + + kfree(mcam->bmap); + kfree(mcam->bmap_reverse); + kfree(mcam->entry2pfvf_map); + kfree(mcam->cntr2pfvf_map); + kfree(mcam->entry2cntr_map); + kfree(mcam->cntr_refcnt); + kfree(mcam->entry2target_pffunc); + kfree(mcam->counters.bmap); +} + +int npc_mcam_rsrcs_init(struct rvu *rvu, int blkaddr) { int nixlf_count = rvu_get_nixlf_count(rvu); struct npc_mcam *mcam = &rvu->hw->mcam; @@ -1890,24 +1904,23 @@ static int npc_mcam_rsrcs_init(struct rvu *rvu, int blkaddr) mcam->pf_offset = mcam->nixlf_offset + nixlf_count; /* Allocate bitmaps for managing MCAM entries */ - mcam->bmap = devm_kcalloc(rvu->dev, BITS_TO_LONGS(mcam->bmap_entries), - sizeof(long), GFP_KERNEL); + mcam->bmap = kmalloc_array(BITS_TO_LONGS(mcam->bmap_entries), + sizeof(long), GFP_KERNEL); if (!mcam->bmap) return -ENOMEM; - mcam->bmap_reverse = devm_kcalloc(rvu->dev, - BITS_TO_LONGS(mcam->bmap_entries), - sizeof(long), GFP_KERNEL); + mcam->bmap_reverse = kmalloc_array(BITS_TO_LONGS(mcam->bmap_entries), + sizeof(long), GFP_KERNEL); if (!mcam->bmap_reverse) - return -ENOMEM; + goto free_bmap; mcam->bmap_fcnt = mcam->bmap_entries; /* Alloc memory for saving entry to RVU PFFUNC allocation mapping */ - mcam->entry2pfvf_map = devm_kcalloc(rvu->dev, mcam->bmap_entries, - sizeof(u16), GFP_KERNEL); + mcam->entry2pfvf_map = kmalloc_array(mcam->bmap_entries, + sizeof(u16), GFP_KERNEL); if (!mcam->entry2pfvf_map) - return -ENOMEM; + goto free_bmap_reverse; /* Reserve 1/8th of MCAM entries at the bottom for low priority * allocations and another 1/8th at the top for high priority @@ -1926,31 +1939,31 @@ static int npc_mcam_rsrcs_init(struct rvu *rvu, int blkaddr) */ err = rvu_alloc_bitmap(&mcam->counters); if (err) - return err; + goto free_entry_map; - mcam->cntr2pfvf_map = devm_kcalloc(rvu->dev, mcam->counters.max, - sizeof(u16), GFP_KERNEL); + mcam->cntr2pfvf_map = kmalloc_array(mcam->counters.max, + sizeof(u16), GFP_KERNEL); if (!mcam->cntr2pfvf_map) - goto free_mem; + goto free_cntr_bmap; /* Alloc memory for MCAM entry to counter mapping and for tracking * counter's reference count. */ - mcam->entry2cntr_map = devm_kcalloc(rvu->dev, mcam->bmap_entries, - sizeof(u16), GFP_KERNEL); + mcam->entry2cntr_map = kmalloc_array(mcam->bmap_entries, + sizeof(u16), GFP_KERNEL); if (!mcam->entry2cntr_map) - goto free_mem; + goto free_cntr_map; - mcam->cntr_refcnt = devm_kcalloc(rvu->dev, mcam->counters.max, - sizeof(u16), GFP_KERNEL); + mcam->cntr_refcnt = kmalloc_array(mcam->counters.max, + sizeof(u16), GFP_KERNEL); if (!mcam->cntr_refcnt) - goto free_mem; + goto free_entry_cntr_map; /* Alloc memory for saving target device of mcam rule */ - mcam->entry2target_pffunc = devm_kcalloc(rvu->dev, mcam->total_entries, - sizeof(u16), GFP_KERNEL); + mcam->entry2target_pffunc = kmalloc_array(mcam->total_entries, + sizeof(u16), GFP_KERNEL); if (!mcam->entry2target_pffunc) - goto free_mem; + goto free_cntr_refcnt; for (index = 0; index < mcam->bmap_entries; index++) { mcam->entry2pfvf_map[index] = NPC_MCAM_INVALID_MAP; @@ -1964,8 +1977,21 @@ static int npc_mcam_rsrcs_init(struct rvu *rvu, int blkaddr) return 0; -free_mem: +free_cntr_refcnt: + kfree(mcam->cntr_refcnt); +free_entry_cntr_map: + kfree(mcam->entry2cntr_map); +free_cntr_map: + kfree(mcam->cntr2pfvf_map); +free_cntr_bmap: + kfree(mcam->counters.bmap); +free_entry_map: + kfree(mcam->entry2pfvf_map); +free_bmap_reverse: + kfree(mcam->bmap_reverse); +free_bmap: kfree(mcam->counters.bmap); + return -ENOMEM; } @@ -2173,6 +2199,7 @@ void rvu_npc_freemem(struct rvu *rvu) struct npc_mcam *mcam = &rvu->hw->mcam; kfree(pkind->rsrc.bmap); + npc_mcam_rsrcs_deinit(rvu); kfree(mcam->counters.bmap); if (rvu->kpu_prfl_addr) iounmap(rvu->kpu_prfl_addr); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index 1b309c47493f4b..0bfe1ca8a36423 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -72,7 +72,6 @@ struct page_pool; #define MLX5E_HW2SW_MTU(params, hwmtu) ((hwmtu) - ((params)->hard_mtu)) #define MLX5E_SW2HW_MTU(params, swmtu) ((swmtu) + ((params)->hard_mtu)) -#define MLX5E_MAX_NUM_TC 8 #define MLX5E_MAX_NUM_MQPRIO_CH_TC TC_QOPT_MAX_QUEUE #define MLX5_RX_HEADROOM NET_SKB_PAD @@ -364,7 +363,7 @@ struct mlx5e_cq { /* control */ struct net_device *netdev; struct mlx5_core_dev *mdev; - struct mlx5e_priv *priv; + struct workqueue_struct *workqueue; struct mlx5_wq_ctrl wq_ctrl; } ____cacheline_aligned_in_smp; @@ -758,7 +757,7 @@ struct mlx5e_channel { /* data path */ struct mlx5e_rq rq; struct mlx5e_xdpsq rq_xdpsq; - struct mlx5e_txqsq sq[MLX5E_MAX_NUM_TC]; + struct mlx5e_txqsq sq[MLX5_MAX_NUM_TC]; struct mlx5e_icosq icosq; /* internal control operations */ struct mlx5e_txqsq __rcu * __rcu *qos_sqs; bool xdp; @@ -808,7 +807,7 @@ struct mlx5e_channels { struct mlx5e_channel_stats { struct mlx5e_ch_stats ch; - struct mlx5e_sq_stats sq[MLX5E_MAX_NUM_TC]; + struct mlx5e_sq_stats sq[MLX5_MAX_NUM_TC]; struct mlx5e_rq_stats rq; struct mlx5e_rq_stats xskrq; struct mlx5e_xdpsq_stats rq_xdpsq; @@ -818,8 +817,8 @@ struct mlx5e_channel_stats { struct mlx5e_ptp_stats { struct mlx5e_ch_stats ch; - struct mlx5e_sq_stats sq[MLX5E_MAX_NUM_TC]; - struct mlx5e_ptp_cq_stats cq[MLX5E_MAX_NUM_TC]; + struct mlx5e_sq_stats sq[MLX5_MAX_NUM_TC]; + struct mlx5e_ptp_cq_stats cq[MLX5_MAX_NUM_TC]; struct mlx5e_rq_stats rq; } ____cacheline_aligned_in_smp; @@ -887,7 +886,6 @@ struct mlx5e_priv { struct mlx5e_rq drop_rq; struct mlx5e_channels channels; - u32 tisn[MLX5_MAX_PORTS][MLX5E_MAX_NUM_TC]; struct mlx5e_rx_res *rx_res; u32 *tx_rates; @@ -985,6 +983,8 @@ struct mlx5e_profile { void (*update_stats)(struct mlx5e_priv *priv); void (*update_carrier)(struct mlx5e_priv *priv); int (*max_nch_limit)(struct mlx5_core_dev *mdev); + u32 (*get_tisn)(struct mlx5_core_dev *mdev, struct mlx5e_priv *priv, + u8 lag_port, u8 tc); unsigned int (*stats_grps_num)(struct mlx5e_priv *priv); mlx5e_stats_grp_t *stats_grps; const struct mlx5e_rx_handlers *rx_handlers; @@ -992,6 +992,11 @@ struct mlx5e_profile { u32 features; }; +u32 mlx5e_profile_get_tisn(struct mlx5_core_dev *mdev, + struct mlx5e_priv *priv, + const struct mlx5e_profile *profile, + u8 lag_port, u8 tc); + #define mlx5e_profile_feature_cap(profile, feature) \ ((profile)->features & BIT(MLX5E_PROFILE_FEATURE_##feature)) @@ -1039,6 +1044,8 @@ int mlx5e_open_xdpsq(struct mlx5e_channel *c, struct mlx5e_params *params, void mlx5e_close_xdpsq(struct mlx5e_xdpsq *sq); struct mlx5e_create_cq_param { + struct net_device *netdev; + struct workqueue_struct *wq; struct napi_struct *napi; struct mlx5e_ch_stats *ch_stats; int node; @@ -1046,7 +1053,7 @@ struct mlx5e_create_cq_param { }; struct mlx5e_cq_param; -int mlx5e_open_cq(struct mlx5e_priv *priv, struct dim_cq_moder moder, +int mlx5e_open_cq(struct mlx5_core_dev *mdev, struct dim_cq_moder moder, struct mlx5e_cq_param *param, struct mlx5e_create_cq_param *ccp, struct mlx5e_cq *cq); void mlx5e_close_cq(struct mlx5e_cq *cq); @@ -1133,8 +1140,6 @@ void mlx5e_close_drop_rq(struct mlx5e_rq *drop_rq); int mlx5e_create_tis(struct mlx5_core_dev *mdev, void *in, u32 *tisn); void mlx5e_destroy_tis(struct mlx5_core_dev *mdev, u32 tisn); -int mlx5e_create_tises(struct mlx5e_priv *priv); -void mlx5e_destroy_tises(struct mlx5e_priv *priv); int mlx5e_update_nic_rx(struct mlx5e_priv *priv); void mlx5e_update_carrier(struct mlx5e_priv *priv); int mlx5e_close(struct net_device *netdev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/monitor_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en/monitor_stats.c index 254c8473904643..40c8df111754fd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/monitor_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/monitor_stats.c @@ -36,7 +36,7 @@ int mlx5e_monitor_counter_supported(struct mlx5e_priv *priv) return true; } -void mlx5e_monitor_counter_arm(struct mlx5e_priv *priv) +static void mlx5e_monitor_counter_arm(struct mlx5e_priv *priv) { u32 in[MLX5_ST_SZ_DW(arm_monitor_counter_in)] = {}; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/monitor_stats.h b/drivers/net/ethernet/mellanox/mlx5/core/en/monitor_stats.h index e1ac4b3d22fbd5..6beba7f075c1b1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/monitor_stats.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/monitor_stats.h @@ -7,6 +7,5 @@ int mlx5e_monitor_counter_supported(struct mlx5e_priv *priv); void mlx5e_monitor_counter_init(struct mlx5e_priv *priv); void mlx5e_monitor_counter_cleanup(struct mlx5e_priv *priv); -void mlx5e_monitor_counter_arm(struct mlx5e_priv *priv); #endif /* __MLX5_MONITOR_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c index e097f336e1c4a0..284253b79266b9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c @@ -669,6 +669,8 @@ void mlx5e_build_rq_params(struct mlx5_core_dev *mdev, void mlx5e_build_create_cq_param(struct mlx5e_create_cq_param *ccp, struct mlx5e_channel *c) { *ccp = (struct mlx5e_create_cq_param) { + .netdev = c->netdev, + .wq = c->priv->wq, .napi = &c->napi, .ch_stats = c->stats, .node = cpu_to_node(c->cpu), diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c index af3928eddafd11..c206cc0a84832e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c @@ -518,9 +518,11 @@ static int mlx5e_ptp_open_txqsqs(struct mlx5e_ptp *c, for (tc = 0; tc < num_tc; tc++) { int txq_ix = ix_base + tc; + u32 tisn; - err = mlx5e_ptp_open_txqsq(c, c->priv->tisn[c->lag_port][tc], txq_ix, - cparams, tc, &c->ptpsq[tc]); + tisn = mlx5e_profile_get_tisn(c->mdev, c->priv, c->priv->profile, + c->lag_port, tc); + err = mlx5e_ptp_open_txqsq(c, tisn, txq_ix, cparams, tc, &c->ptpsq[tc]); if (err) goto close_txqsq; } @@ -555,6 +557,8 @@ static int mlx5e_ptp_open_tx_cqs(struct mlx5e_ptp *c, num_tc = mlx5e_get_dcb_num_tc(params); + ccp.netdev = c->netdev; + ccp.wq = c->priv->wq; ccp.node = dev_to_node(mlx5_core_dma_dev(c->mdev)); ccp.ch_stats = c->stats; ccp.napi = &c->napi; @@ -565,7 +569,7 @@ static int mlx5e_ptp_open_tx_cqs(struct mlx5e_ptp *c, for (tc = 0; tc < num_tc; tc++) { struct mlx5e_cq *cq = &c->ptpsq[tc].txqsq.cq; - err = mlx5e_open_cq(c->priv, ptp_moder, cq_param, &ccp, cq); + err = mlx5e_open_cq(c->mdev, ptp_moder, cq_param, &ccp, cq); if (err) goto out_err_txqsq_cq; } @@ -574,7 +578,7 @@ static int mlx5e_ptp_open_tx_cqs(struct mlx5e_ptp *c, struct mlx5e_cq *cq = &c->ptpsq[tc].ts_cq; struct mlx5e_ptpsq *ptpsq = &c->ptpsq[tc]; - err = mlx5e_open_cq(c->priv, ptp_moder, cq_param, &ccp, cq); + err = mlx5e_open_cq(c->mdev, ptp_moder, cq_param, &ccp, cq); if (err) goto out_err_ts_cq; @@ -602,6 +606,8 @@ static int mlx5e_ptp_open_rx_cq(struct mlx5e_ptp *c, struct mlx5e_cq_param *cq_param; struct mlx5e_cq *cq = &c->rq.cq; + ccp.netdev = c->netdev; + ccp.wq = c->priv->wq; ccp.node = dev_to_node(mlx5_core_dma_dev(c->mdev)); ccp.ch_stats = c->stats; ccp.napi = &c->napi; @@ -609,7 +615,7 @@ static int mlx5e_ptp_open_rx_cq(struct mlx5e_ptp *c, cq_param = &cparams->rq_param.cqp; - return mlx5e_open_cq(c->priv, ptp_moder, cq_param, &ccp, cq); + return mlx5e_open_cq(c->mdev, ptp_moder, cq_param, &ccp, cq); } static void mlx5e_ptp_close_tx_cqs(struct mlx5e_ptp *c) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h index 7b700d0f956a88..86f1854698b4e8 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h @@ -49,7 +49,7 @@ enum { struct mlx5e_ptp { /* data path */ - struct mlx5e_ptpsq ptpsq[MLX5E_MAX_NUM_TC]; + struct mlx5e_ptpsq ptpsq[MLX5_MAX_NUM_TC]; struct mlx5e_rq rq; struct napi_struct napi; struct device *pdev; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c b/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c index 244bc15a42abff..34adf8c3f81a08 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c @@ -77,6 +77,7 @@ int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs, struct mlx5e_params *params; struct mlx5e_channel *c; struct mlx5e_txqsq *sq; + u32 tisn; params = &chs->params; @@ -123,11 +124,13 @@ int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs, memset(¶m_cq, 0, sizeof(param_cq)); mlx5e_build_sq_param(priv->mdev, params, ¶m_sq); mlx5e_build_tx_cq_param(priv->mdev, params, ¶m_cq); - err = mlx5e_open_cq(priv, params->tx_cq_moderation, ¶m_cq, &ccp, &sq->cq); + err = mlx5e_open_cq(c->mdev, params->tx_cq_moderation, ¶m_cq, &ccp, &sq->cq); if (err) goto err_free_sq; - err = mlx5e_open_txqsq(c, priv->tisn[c->lag_port][0], txq_ix, params, - ¶m_sq, sq, 0, hw_id, + + tisn = mlx5e_profile_get_tisn(c->mdev, c->priv, c->priv->profile, + c->lag_port, 0); + err = mlx5e_open_txqsq(c, tisn, txq_ix, params, ¶m_sq, sq, 0, hw_id, priv->htb_qos_sq_stats[node_qid]); if (err) goto err_close_cq; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/trap.c b/drivers/net/ethernet/mellanox/mlx5/core/en/trap.c index 5620d9f9751808..ac458a8d10e0f1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/trap.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/trap.c @@ -68,11 +68,13 @@ static int mlx5e_open_trap_rq(struct mlx5e_priv *priv, struct mlx5e_trap *t) node = dev_to_node(mdev->device); + ccp.netdev = priv->netdev; + ccp.wq = priv->wq; ccp.node = node; ccp.ch_stats = t->stats; ccp.napi = &t->napi; ccp.ix = 0; - err = mlx5e_open_cq(priv, trap_moder, &rq_param->cqp, &ccp, &rq->cq); + err = mlx5e_open_cq(priv->mdev, trap_moder, &rq_param->cqp, &ccp, &rq->cq); if (err) return err; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c index e8564fb7f9a080..82b5ca1be4f395 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c @@ -256,9 +256,24 @@ static int mlx5e_xdp_rx_hash(const struct xdp_md *ctx, u32 *hash, return 0; } +static int mlx5e_xdp_rx_vlan_tag(const struct xdp_md *ctx, __be16 *vlan_proto, + u16 *vlan_tci) +{ + const struct mlx5e_xdp_buff *_ctx = (void *)ctx; + const struct mlx5_cqe64 *cqe = _ctx->cqe; + + if (!cqe_has_vlan(cqe)) + return -ENODATA; + + *vlan_proto = htons(ETH_P_8021Q); + *vlan_tci = be16_to_cpu(cqe->vlan_info); + return 0; +} + const struct xdp_metadata_ops mlx5e_xdp_metadata_ops = { .xmo_rx_timestamp = mlx5e_xdp_rx_timestamp, .xmo_rx_hash = mlx5e_xdp_rx_hash, + .xmo_rx_vlan_tag = mlx5e_xdp_rx_vlan_tag, }; struct mlx5e_xsk_tx_complete { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c index 36826b58248478..82e6abbc173423 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c @@ -127,7 +127,7 @@ int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params, mlx5e_build_xsk_cparam(priv->mdev, params, xsk, priv->q_counter, cparam); - err = mlx5e_open_cq(c->priv, params->rx_cq_moderation, &cparam->rq.cqp, &ccp, + err = mlx5e_open_cq(c->mdev, params->rx_cq_moderation, &cparam->rq.cqp, &ccp, &c->xskrq.cq); if (unlikely(err)) goto err_free_cparam; @@ -136,7 +136,7 @@ int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params, if (unlikely(err)) goto err_close_rx_cq; - err = mlx5e_open_cq(c->priv, params->tx_cq_moderation, &cparam->xdp_sq.cqp, &ccp, + err = mlx5e_open_cq(c->mdev, params->tx_cq_moderation, &cparam->xdp_sq.cqp, &ccp, &c->xsksq.cq); if (unlikely(err)) goto err_close_rq; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_common.c b/drivers/net/ethernet/mellanox/mlx5/core/en_common.c index 41c396e7645794..67f546683e85a3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_common.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_common.c @@ -74,6 +74,72 @@ int mlx5e_create_mkey(struct mlx5_core_dev *mdev, u32 pdn, u32 *mkey) return err; } +int mlx5e_create_tis(struct mlx5_core_dev *mdev, void *in, u32 *tisn) +{ + void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx); + + MLX5_SET(tisc, tisc, transport_domain, mdev->mlx5e_res.hw_objs.td.tdn); + + if (mlx5_lag_is_lacp_owner(mdev)) + MLX5_SET(tisc, tisc, strict_lag_tx_port_affinity, 1); + + return mlx5_core_create_tis(mdev, in, tisn); +} + +void mlx5e_destroy_tis(struct mlx5_core_dev *mdev, u32 tisn) +{ + mlx5_core_destroy_tis(mdev, tisn); +} + +static void mlx5e_destroy_tises(struct mlx5_core_dev *mdev, u32 tisn[MLX5_MAX_PORTS][MLX5_MAX_NUM_TC]) +{ + int tc, i; + + for (i = 0; i < MLX5_MAX_PORTS; i++) + for (tc = 0; tc < MLX5_MAX_NUM_TC; tc++) + mlx5e_destroy_tis(mdev, tisn[i][tc]); +} + +static bool mlx5_lag_should_assign_affinity(struct mlx5_core_dev *mdev) +{ + return MLX5_CAP_GEN(mdev, lag_tx_port_affinity) && mlx5e_get_num_lag_ports(mdev) > 1; +} + +static int mlx5e_create_tises(struct mlx5_core_dev *mdev, u32 tisn[MLX5_MAX_PORTS][MLX5_MAX_NUM_TC]) +{ + int tc, i; + int err; + + for (i = 0; i < MLX5_MAX_PORTS; i++) { + for (tc = 0; tc < MLX5_MAX_NUM_TC; tc++) { + u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {}; + void *tisc; + + tisc = MLX5_ADDR_OF(create_tis_in, in, ctx); + + MLX5_SET(tisc, tisc, prio, tc << 1); + + if (mlx5_lag_should_assign_affinity(mdev)) + MLX5_SET(tisc, tisc, lag_tx_port_affinity, i + 1); + + err = mlx5e_create_tis(mdev, in, &tisn[i][tc]); + if (err) + goto err_close_tises; + } + } + + return 0; + +err_close_tises: + for (; i >= 0; i--) { + for (tc--; tc >= 0; tc--) + mlx5e_destroy_tis(mdev, tisn[i][tc]); + tc = MLX5_MAX_NUM_TC; + } + + return err; +} + int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev) { struct mlx5e_hw_objs *res = &mdev->mlx5e_res.hw_objs; @@ -103,6 +169,11 @@ int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev) goto err_destroy_mkey; } + err = mlx5e_create_tises(mdev, res->tisn); + if (err) { + mlx5_core_err(mdev, "alloc tises failed, %d\n", err); + goto err_destroy_bfreg; + } INIT_LIST_HEAD(&res->td.tirs_list); mutex_init(&res->td.list_lock); @@ -115,6 +186,8 @@ int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev) return 0; +err_destroy_bfreg: + mlx5_free_bfreg(mdev, &res->bfreg); err_destroy_mkey: mlx5_core_destroy_mkey(mdev, res->mkey); err_dealloc_transport_domain: @@ -130,6 +203,7 @@ void mlx5e_destroy_mdev_resources(struct mlx5_core_dev *mdev) mlx5_crypto_dek_cleanup(mdev->mlx5e_res.dek_priv); mdev->mlx5e_res.dek_priv = NULL; + mlx5e_destroy_tises(mdev, res->tisn); mlx5_free_bfreg(mdev, &res->bfreg); mlx5_core_destroy_mkey(mdev, res->mkey); mlx5_core_dealloc_transport_domain(mdev, res->td.tdn); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 82eddfb19efcba..b5f1c4ca38bac9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -1352,6 +1352,17 @@ void mlx5e_close_rq(struct mlx5e_rq *rq) mlx5e_free_rq(rq); } +u32 mlx5e_profile_get_tisn(struct mlx5_core_dev *mdev, + struct mlx5e_priv *priv, + const struct mlx5e_profile *profile, + u8 lag_port, u8 tc) +{ + if (profile->get_tisn) + return profile->get_tisn(mdev, priv, lag_port, tc); + + return mdev->mlx5e_res.hw_objs.tisn[lag_port][tc]; +} + static void mlx5e_free_xdpsq_db(struct mlx5e_xdpsq *sq) { kvfree(sq->db.xdpi_fifo.xi); @@ -1920,7 +1931,8 @@ int mlx5e_open_xdpsq(struct mlx5e_channel *c, struct mlx5e_params *params, return err; csp.tis_lst_sz = 1; - csp.tisn = c->priv->tisn[c->lag_port][0]; /* tc = 0 */ + csp.tisn = mlx5e_profile_get_tisn(c->mdev, c->priv, c->priv->profile, + c->lag_port, 0); /* tc = 0 */ csp.cqn = sq->cq.mcq.cqn; csp.wq_ctrl = &sq->wq_ctrl; csp.min_inline_mode = sq->min_inline_mode; @@ -1982,11 +1994,12 @@ void mlx5e_close_xdpsq(struct mlx5e_xdpsq *sq) mlx5e_free_xdpsq(sq); } -static int mlx5e_alloc_cq_common(struct mlx5e_priv *priv, +static int mlx5e_alloc_cq_common(struct mlx5_core_dev *mdev, + struct net_device *netdev, + struct workqueue_struct *workqueue, struct mlx5e_cq_param *param, struct mlx5e_cq *cq) { - struct mlx5_core_dev *mdev = priv->mdev; struct mlx5_core_cq *mcq = &cq->mcq; int err; u32 i; @@ -2013,13 +2026,13 @@ static int mlx5e_alloc_cq_common(struct mlx5e_priv *priv, } cq->mdev = mdev; - cq->netdev = priv->netdev; - cq->priv = priv; + cq->netdev = netdev; + cq->workqueue = workqueue; return 0; } -static int mlx5e_alloc_cq(struct mlx5e_priv *priv, +static int mlx5e_alloc_cq(struct mlx5_core_dev *mdev, struct mlx5e_cq_param *param, struct mlx5e_create_cq_param *ccp, struct mlx5e_cq *cq) @@ -2030,7 +2043,7 @@ static int mlx5e_alloc_cq(struct mlx5e_priv *priv, param->wq.db_numa_node = ccp->node; param->eq_ix = ccp->ix; - err = mlx5e_alloc_cq_common(priv, param, cq); + err = mlx5e_alloc_cq_common(mdev, ccp->netdev, ccp->wq, param, cq); cq->napi = ccp->napi; cq->ch_stats = ccp->ch_stats; @@ -2096,14 +2109,13 @@ static void mlx5e_destroy_cq(struct mlx5e_cq *cq) mlx5_core_destroy_cq(cq->mdev, &cq->mcq); } -int mlx5e_open_cq(struct mlx5e_priv *priv, struct dim_cq_moder moder, +int mlx5e_open_cq(struct mlx5_core_dev *mdev, struct dim_cq_moder moder, struct mlx5e_cq_param *param, struct mlx5e_create_cq_param *ccp, struct mlx5e_cq *cq) { - struct mlx5_core_dev *mdev = priv->mdev; int err; - err = mlx5e_alloc_cq(priv, param, ccp, cq); + err = mlx5e_alloc_cq(mdev, param, ccp, cq); if (err) return err; @@ -2136,7 +2148,7 @@ static int mlx5e_open_tx_cqs(struct mlx5e_channel *c, int tc; for (tc = 0; tc < c->num_tc; tc++) { - err = mlx5e_open_cq(c->priv, params->tx_cq_moderation, &cparam->txq_sq.cqp, + err = mlx5e_open_cq(c->mdev, params->tx_cq_moderation, &cparam->txq_sq.cqp, ccp, &c->sq[tc].cq); if (err) goto err_close_tx_cqs; @@ -2204,12 +2216,15 @@ static int mlx5e_open_sqs(struct mlx5e_channel *c, for (tc = 0; tc < mlx5e_get_dcb_num_tc(params); tc++) { int txq_ix = c->ix + tc * params->num_channels; u32 qos_queue_group_id; + u32 tisn; + tisn = mlx5e_profile_get_tisn(c->mdev, c->priv, c->priv->profile, + c->lag_port, tc); err = mlx5e_txq_get_qos_node_hw_id(params, txq_ix, &qos_queue_group_id); if (err) goto err_close_sqs; - err = mlx5e_open_txqsq(c, c->priv->tisn[c->lag_port][tc], txq_ix, + err = mlx5e_open_txqsq(c, tisn, txq_ix, params, &cparam->txq_sq, &c->sq[tc], tc, qos_queue_group_id, &c->priv->channel_stats[c->ix]->sq[tc]); @@ -2337,12 +2352,12 @@ static int mlx5e_open_queues(struct mlx5e_channel *c, mlx5e_build_create_cq_param(&ccp, c); - err = mlx5e_open_cq(c->priv, icocq_moder, &cparam->async_icosq.cqp, &ccp, + err = mlx5e_open_cq(c->mdev, icocq_moder, &cparam->async_icosq.cqp, &ccp, &c->async_icosq.cq); if (err) return err; - err = mlx5e_open_cq(c->priv, icocq_moder, &cparam->icosq.cqp, &ccp, + err = mlx5e_open_cq(c->mdev, icocq_moder, &cparam->icosq.cqp, &ccp, &c->icosq.cq); if (err) goto err_close_async_icosq_cq; @@ -2351,17 +2366,17 @@ static int mlx5e_open_queues(struct mlx5e_channel *c, if (err) goto err_close_icosq_cq; - err = mlx5e_open_cq(c->priv, params->tx_cq_moderation, &cparam->xdp_sq.cqp, &ccp, + err = mlx5e_open_cq(c->mdev, params->tx_cq_moderation, &cparam->xdp_sq.cqp, &ccp, &c->xdpsq.cq); if (err) goto err_close_tx_cqs; - err = mlx5e_open_cq(c->priv, params->rx_cq_moderation, &cparam->rq.cqp, &ccp, + err = mlx5e_open_cq(c->mdev, params->rx_cq_moderation, &cparam->rq.cqp, &ccp, &c->rq.cq); if (err) goto err_close_xdp_tx_cqs; - err = c->xdp ? mlx5e_open_cq(c->priv, params->tx_cq_moderation, &cparam->xdp_sq.cqp, + err = c->xdp ? mlx5e_open_cq(c->mdev, params->tx_cq_moderation, &cparam->xdp_sq.cqp, &ccp, &c->rq_xdpsq.cq) : 0; if (err) goto err_close_rx_cq; @@ -3308,7 +3323,7 @@ static int mlx5e_alloc_drop_cq(struct mlx5e_priv *priv, param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev)); param->wq.db_numa_node = dev_to_node(mlx5_core_dma_dev(mdev)); - return mlx5e_alloc_cq_common(priv, param, cq); + return mlx5e_alloc_cq_common(priv->mdev, priv->netdev, priv->wq, param, cq); } int mlx5e_open_drop_rq(struct mlx5e_priv *priv, @@ -3364,75 +3379,6 @@ void mlx5e_close_drop_rq(struct mlx5e_rq *drop_rq) mlx5e_free_cq(&drop_rq->cq); } -int mlx5e_create_tis(struct mlx5_core_dev *mdev, void *in, u32 *tisn) -{ - void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx); - - MLX5_SET(tisc, tisc, transport_domain, mdev->mlx5e_res.hw_objs.td.tdn); - - if (MLX5_GET(tisc, tisc, tls_en)) - MLX5_SET(tisc, tisc, pd, mdev->mlx5e_res.hw_objs.pdn); - - if (mlx5_lag_is_lacp_owner(mdev)) - MLX5_SET(tisc, tisc, strict_lag_tx_port_affinity, 1); - - return mlx5_core_create_tis(mdev, in, tisn); -} - -void mlx5e_destroy_tis(struct mlx5_core_dev *mdev, u32 tisn) -{ - mlx5_core_destroy_tis(mdev, tisn); -} - -void mlx5e_destroy_tises(struct mlx5e_priv *priv) -{ - int tc, i; - - for (i = 0; i < mlx5e_get_num_lag_ports(priv->mdev); i++) - for (tc = 0; tc < priv->profile->max_tc; tc++) - mlx5e_destroy_tis(priv->mdev, priv->tisn[i][tc]); -} - -static bool mlx5e_lag_should_assign_affinity(struct mlx5_core_dev *mdev) -{ - return MLX5_CAP_GEN(mdev, lag_tx_port_affinity) && mlx5e_get_num_lag_ports(mdev) > 1; -} - -int mlx5e_create_tises(struct mlx5e_priv *priv) -{ - int tc, i; - int err; - - for (i = 0; i < mlx5e_get_num_lag_ports(priv->mdev); i++) { - for (tc = 0; tc < priv->profile->max_tc; tc++) { - u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {}; - void *tisc; - - tisc = MLX5_ADDR_OF(create_tis_in, in, ctx); - - MLX5_SET(tisc, tisc, prio, tc << 1); - - if (mlx5e_lag_should_assign_affinity(priv->mdev)) - MLX5_SET(tisc, tisc, lag_tx_port_affinity, i + 1); - - err = mlx5e_create_tis(priv->mdev, in, &priv->tisn[i][tc]); - if (err) - goto err_close_tises; - } - } - - return 0; - -err_close_tises: - for (; i >= 0; i--) { - for (tc--; tc >= 0; tc--) - mlx5e_destroy_tis(priv->mdev, priv->tisn[i][tc]); - tc = priv->profile->max_tc; - } - - return err; -} - static void mlx5e_cleanup_nic_tx(struct mlx5e_priv *priv) { if (priv->mqprio_rl) { @@ -3441,7 +3387,6 @@ static void mlx5e_cleanup_nic_tx(struct mlx5e_priv *priv) priv->mqprio_rl = NULL; } mlx5e_accel_cleanup_tx(priv); - mlx5e_destroy_tises(priv); } static int mlx5e_modify_channels_vsd(struct mlx5e_channels *chs, bool vsd) @@ -3543,7 +3488,7 @@ static int mlx5e_setup_tc_mqprio_dcb(struct mlx5e_priv *priv, mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; - if (tc && tc != MLX5E_MAX_NUM_TC) + if (tc && tc != MLX5_MAX_NUM_TC) return -EINVAL; new_params = priv->channels.params; @@ -5506,23 +5451,13 @@ static int mlx5e_init_nic_tx(struct mlx5e_priv *priv) { int err; - err = mlx5e_create_tises(priv); - if (err) { - mlx5_core_warn(priv->mdev, "create tises failed, %d\n", err); - return err; - } - err = mlx5e_accel_init_tx(priv); if (err) - goto err_destroy_tises; + return err; mlx5e_set_mqprio_rl(priv); mlx5e_dcbnl_initialize(priv); return 0; - -err_destroy_tises: - mlx5e_destroy_tises(priv); - return err; } static void mlx5e_nic_enable(struct mlx5e_priv *priv) @@ -5617,7 +5552,7 @@ static const struct mlx5e_profile mlx5e_nic_profile = { .update_stats = mlx5e_stats_update_ndo_stats, .update_carrier = mlx5e_update_carrier, .rx_handlers = &mlx5e_rx_handlers_nic, - .max_tc = MLX5E_MAX_NUM_TC, + .max_tc = MLX5_MAX_NUM_TC, .stats_grps = mlx5e_nic_stats_grps, .stats_grps_num = mlx5e_nic_stats_grps_num, .features = BIT(MLX5E_PROFILE_FEATURE_PTP_RX) | @@ -6070,7 +6005,7 @@ static int mlx5e_resume(struct auxiliary_device *adev) return 0; } -static int mlx5e_suspend(struct auxiliary_device *adev, pm_message_t state) +static int _mlx5e_suspend(struct auxiliary_device *adev) { struct mlx5e_dev *mlx5e_dev = auxiliary_get_drvdata(adev); struct mlx5e_priv *priv = mlx5e_dev->priv; @@ -6088,15 +6023,18 @@ static int mlx5e_suspend(struct auxiliary_device *adev, pm_message_t state) return 0; } -static int mlx5e_probe(struct auxiliary_device *adev, - const struct auxiliary_device_id *id) +static int mlx5e_suspend(struct auxiliary_device *adev, pm_message_t state) +{ + return _mlx5e_suspend(adev); +} + +static int _mlx5e_probe(struct auxiliary_device *adev) { struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); const struct mlx5e_profile *profile = &mlx5e_nic_profile; struct mlx5_core_dev *mdev = edev->mdev; struct mlx5e_dev *mlx5e_dev; struct net_device *netdev; - pm_message_t state = {}; struct mlx5e_priv *priv; int err; @@ -6151,7 +6089,7 @@ static int mlx5e_probe(struct auxiliary_device *adev, return 0; err_resume: - mlx5e_suspend(adev, state); + _mlx5e_suspend(adev); err_profile_cleanup: profile->cleanup(priv); err_destroy_netdev: @@ -6163,16 +6101,21 @@ static int mlx5e_probe(struct auxiliary_device *adev, return err; } +static int mlx5e_probe(struct auxiliary_device *adev, + const struct auxiliary_device_id *id) +{ + return _mlx5e_probe(adev); +} + static void mlx5e_remove(struct auxiliary_device *adev) { struct mlx5e_dev *mlx5e_dev = auxiliary_get_drvdata(adev); struct mlx5e_priv *priv = mlx5e_dev->priv; - pm_message_t state = {}; mlx5_core_uplink_netdev_set(priv->mdev, NULL); mlx5e_dcbnl_delete_app(priv); unregister_netdev(priv->netdev); - mlx5e_suspend(adev, state); + _mlx5e_suspend(adev); priv->profile->cleanup(priv); mlx5e_destroy_netdev(priv); mlx5e_devlink_port_unregister(mlx5e_dev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c index f49bf2f644e1c2..05527418fa642f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c @@ -1180,12 +1180,6 @@ static int mlx5e_init_rep_tx(struct mlx5e_priv *priv) struct mlx5e_rep_priv *rpriv = priv->ppriv; int err; - err = mlx5e_create_tises(priv); - if (err) { - mlx5_core_warn(priv->mdev, "create tises failed, %d\n", err); - return err; - } - err = mlx5e_rep_neigh_init(rpriv); if (err) goto err_neigh_init; @@ -1208,7 +1202,6 @@ static int mlx5e_init_rep_tx(struct mlx5e_priv *priv) err_init_tx: mlx5e_rep_neigh_cleanup(rpriv); err_neigh_init: - mlx5e_destroy_tises(priv); return err; } @@ -1222,7 +1215,6 @@ static void mlx5e_cleanup_rep_tx(struct mlx5e_priv *priv) mlx5e_cleanup_uplink_rep_tx(rpriv); mlx5e_rep_neigh_cleanup(rpriv); - mlx5e_destroy_tises(priv); } static void mlx5e_rep_enable(struct mlx5e_priv *priv) @@ -1452,7 +1444,7 @@ static const struct mlx5e_profile mlx5e_uplink_rep_profile = { .update_stats = mlx5e_stats_update_ndo_stats, .update_carrier = mlx5e_update_carrier, .rx_handlers = &mlx5e_rx_handlers_rep, - .max_tc = MLX5E_MAX_NUM_TC, + .max_tc = MLX5_MAX_NUM_TC, .stats_grps = mlx5e_ul_rep_stats_grps, .stats_grps_num = mlx5e_ul_rep_stats_grps_num, }; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index 98d33ac7ec6468..d601b5faaed5b8 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -1039,7 +1039,7 @@ int mlx5e_poll_ico_cq(struct mlx5e_cq *cq) (struct mlx5_err_cqe *)cqe); mlx5_wq_cyc_wqe_dump(&sq->wq, ci, wi->num_wqebbs); if (!test_and_set_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state)) - queue_work(cq->priv->wq, &sq->recover_work); + queue_work(cq->workqueue, &sq->recover_work); break; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c index f0b506e562df31..5c166d9d2dca62 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c @@ -861,7 +861,7 @@ bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget) mlx5e_dump_error_cqe(&sq->cq, sq->sqn, (struct mlx5_err_cqe *)cqe); mlx5_wq_cyc_wqe_dump(&sq->wq, ci, wi->num_wqebbs); - queue_work(cq->priv->wq, &sq->recover_work); + queue_work(cq->workqueue, &sq->recover_work); } stats->cqe_err++; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c index a4b92533166182..1616a6144f7b42 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c @@ -1144,3 +1144,37 @@ const struct mlx5_flow_cmds *mlx5_fs_cmd_get_default(enum fs_flow_table_type typ return mlx5_fs_cmd_get_stub_cmds(); } } + +int mlx5_fs_cmd_set_l2table_entry_silent(struct mlx5_core_dev *dev, u8 silent_mode) +{ + u32 in[MLX5_ST_SZ_DW(set_l2_table_entry_in)] = {}; + + if (silent_mode && !MLX5_CAP_GEN(dev, silent_mode)) + return -EOPNOTSUPP; + + MLX5_SET(set_l2_table_entry_in, in, opcode, MLX5_CMD_OP_SET_L2_TABLE_ENTRY); + MLX5_SET(set_l2_table_entry_in, in, silent_mode_valid, 1); + MLX5_SET(set_l2_table_entry_in, in, silent_mode, silent_mode); + + return mlx5_cmd_exec_in(dev, set_l2_table_entry, in); +} + +int mlx5_fs_cmd_set_tx_flow_table_root(struct mlx5_core_dev *dev, u32 ft_id, bool disconnect) +{ + u32 out[MLX5_ST_SZ_DW(set_flow_table_root_out)] = {}; + u32 in[MLX5_ST_SZ_DW(set_flow_table_root_in)] = {}; + + if (disconnect && MLX5_CAP_FLOWTABLE_NIC_TX(dev, reset_root_to_default)) + return -EOPNOTSUPP; + + MLX5_SET(set_flow_table_root_in, in, opcode, + MLX5_CMD_OP_SET_FLOW_TABLE_ROOT); + MLX5_SET(set_flow_table_root_in, in, table_type, + FS_FT_NIC_TX); + if (disconnect) + MLX5_SET(set_flow_table_root_in, in, op_mod, 1); + else + MLX5_SET(set_flow_table_root_in, in, table_id, ft_id); + + return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.h b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.h index 7790ae5531e1d3..53e0e5137d3ff0 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.h @@ -122,4 +122,6 @@ int mlx5_cmd_fc_bulk_query(struct mlx5_core_dev *dev, u32 base_id, int bulk_len, const struct mlx5_flow_cmds *mlx5_fs_cmd_get_default(enum fs_flow_table_type type); const struct mlx5_flow_cmds *mlx5_fs_cmd_get_fw_cmds(void); +int mlx5_fs_cmd_set_l2table_entry_silent(struct mlx5_core_dev *dev, u8 silent_mode); +int mlx5_fs_cmd_set_tx_flow_table_root(struct mlx5_core_dev *dev, u32 ft_id, bool disconnect); #endif diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c index 2bf77a5251b40a..58845121954c19 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c @@ -339,7 +339,7 @@ static int mlx5i_init_tx(struct mlx5e_priv *priv) return err; } - err = mlx5i_create_tis(priv->mdev, ipriv->qpn, &priv->tisn[0][0]); + err = mlx5i_create_tis(priv->mdev, ipriv->qpn, &ipriv->tisn); if (err) { mlx5_core_warn(priv->mdev, "create tis failed, %d\n", err); goto err_destroy_underlay_qp; @@ -356,7 +356,7 @@ static void mlx5i_cleanup_tx(struct mlx5e_priv *priv) { struct mlx5i_priv *ipriv = priv->ppriv; - mlx5e_destroy_tis(priv->mdev, priv->tisn[0][0]); + mlx5e_destroy_tis(priv->mdev, ipriv->tisn); mlx5i_destroy_underlay_qp(priv->mdev, ipriv->qpn); } @@ -483,6 +483,18 @@ static unsigned int mlx5i_stats_grps_num(struct mlx5e_priv *priv) return ARRAY_SIZE(mlx5i_stats_grps); } +u32 mlx5i_get_tisn(struct mlx5_core_dev *mdev, struct mlx5e_priv *priv, u8 lag_port, u8 tc) +{ + struct mlx5i_priv *ipriv = priv->ppriv; + + if (WARN(lag_port || tc, + "IPoIB unexpected non-zero value: lag_port (%u), tc (%u)\n", + lag_port, tc)) + return 0; + + return ipriv->tisn; +} + static const struct mlx5e_profile mlx5i_nic_profile = { .init = mlx5i_init, .cleanup = mlx5i_cleanup, @@ -499,6 +511,7 @@ static const struct mlx5e_profile mlx5i_nic_profile = { .max_tc = MLX5I_MAX_NUM_TC, .stats_grps = mlx5i_stats_grps, .stats_grps_num = mlx5i_stats_grps_num, + .get_tisn = mlx5i_get_tisn, }; /* mlx5i netdev NDos */ @@ -829,7 +842,7 @@ int mlx5_rdma_rn_get_params(struct mlx5_core_dev *mdev, *params = (struct rdma_netdev_alloc_params){ .sizeof_priv = sizeof(struct mlx5i_priv) + sizeof(struct mlx5e_priv), - .txqs = nch * MLX5E_MAX_NUM_TC, + .txqs = nch * MLX5_MAX_NUM_TC, .rxqs = nch, .param = mdev, .initialize_rdma_netdev = mlx5_rdma_setup_rn, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.h b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.h index f3f2af972020af..2ab6437a1c49fc 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.h @@ -53,6 +53,7 @@ extern const struct mlx5e_rx_handlers mlx5i_rx_handlers; struct mlx5i_priv { struct rdma_netdev rn; /* keep this first */ u32 qpn; + u32 tisn; bool sub_interface; u32 num_sub_interfaces; u32 qkey; @@ -63,6 +64,7 @@ struct mlx5i_priv { }; int mlx5i_create_tis(struct mlx5_core_dev *mdev, u32 underlay_qpn, u32 *tisn); +u32 mlx5i_get_tisn(struct mlx5_core_dev *mdev, struct mlx5e_priv *priv, u8 lag_port, u8 tc); /* Underlay QP create/destroy functions */ int mlx5i_create_underlay_qp(struct mlx5e_priv *priv); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c index 03e681297937f8..f87471306f6b54 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c @@ -218,7 +218,7 @@ static int mlx5i_pkey_open(struct net_device *netdev) goto err_unint_underlay_qp; } - err = mlx5i_create_tis(mdev, ipriv->qpn, &epriv->tisn[0][0]); + err = mlx5i_create_tis(mdev, ipriv->qpn, &ipriv->tisn); if (err) { mlx5_core_warn(mdev, "create child tis failed, %d\n", err); goto err_remove_rx_uderlay_qp; @@ -240,7 +240,7 @@ static int mlx5i_pkey_open(struct net_device *netdev) err_close_channels: mlx5e_close_channels(&epriv->channels); err_clear_state_opened_flag: - mlx5e_destroy_tis(mdev, epriv->tisn[0][0]); + mlx5e_destroy_tis(mdev, ipriv->tisn); err_remove_rx_uderlay_qp: mlx5_fs_remove_rx_underlay_qpn(mdev, ipriv->qpn); err_unint_underlay_qp: @@ -269,7 +269,7 @@ static int mlx5i_pkey_close(struct net_device *netdev) mlx5i_uninit_underlay_qp(priv); mlx5e_deactivate_priv_channels(priv); mlx5e_close_channels(&priv->channels); - mlx5e_destroy_tis(mdev, priv->tisn[0][0]); + mlx5e_destroy_tis(mdev, ipriv->tisn); unlock: mutex_unlock(&priv->state_lock); return 0; @@ -361,6 +361,7 @@ static const struct mlx5e_profile mlx5i_pkey_nic_profile = { .update_stats = NULL, .rx_handlers = &mlx5i_rx_handlers, .max_tc = MLX5I_MAX_NUM_TC, + .get_tisn = mlx5i_get_tisn, }; const struct mlx5e_profile *mlx5i_pkey_get_profile(void) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c index e8e50563e95629..e7d59cfa8708e1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c @@ -256,6 +256,13 @@ void mlx5_devcom_unregister_component(struct mlx5_devcom_comp_dev *devcom) devcom_free_comp_dev(devcom); } +int mlx5_devcom_comp_get_size(struct mlx5_devcom_comp_dev *devcom) +{ + struct mlx5_devcom_comp *comp = devcom->comp; + + return kref_read(&comp->ref); +} + int mlx5_devcom_send_event(struct mlx5_devcom_comp_dev *devcom, int event, int rollback_event, void *event_data) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h index fc23bbef87b461..ec32b686f58654 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h @@ -31,6 +31,7 @@ void mlx5_devcom_unregister_component(struct mlx5_devcom_comp_dev *devcom); int mlx5_devcom_send_event(struct mlx5_devcom_comp_dev *devcom, int event, int rollback_event, void *event_data); +int mlx5_devcom_comp_get_size(struct mlx5_devcom_comp_dev *devcom); void mlx5_devcom_comp_set_ready(struct mlx5_devcom_comp_dev *devcom, bool ready); bool mlx5_devcom_comp_is_ready(struct mlx5_devcom_comp_dev *devcom); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h index 6b14e347d91420..a79b7959361b5c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h @@ -243,6 +243,7 @@ int mlx5_query_mcam_reg(struct mlx5_core_dev *dev, u32 *mcap, u8 feature_group, u8 access_reg_group); int mlx5_query_qcam_reg(struct mlx5_core_dev *mdev, u32 *qcam, u8 feature_group, u8 access_reg_group); +int mlx5_query_mpir_reg(struct mlx5_core_dev *dev, u32 *mpir); void mlx5_lag_add_netdev(struct mlx5_core_dev *dev, struct net_device *netdev); void mlx5_lag_remove_netdev(struct mlx5_core_dev *dev, struct net_device *netdev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/port.c b/drivers/net/ethernet/mellanox/mlx5/core/port.c index 7d8c732818f201..7fba1c46e2ac4a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/port.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/port.c @@ -1206,3 +1206,13 @@ int mlx5_port_max_linkspeed(struct mlx5_core_dev *mdev, u32 *speed) *speed = max_speed; return 0; } + +int mlx5_query_mpir_reg(struct mlx5_core_dev *dev, u32 *mpir) +{ + u32 in[MLX5_ST_SZ_DW(mpir_reg)] = {}; + int sz = MLX5_ST_SZ_BYTES(mpir_reg); + + MLX5_SET(mpir_reg, in, local_port, 1); + + return mlx5_core_access_reg(dev, in, sz, mpir, sz, MLX5_REG_MPIR, 0, 0); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_action.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_action.c index e3ec559369fa07..6f9790e97fed20 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_action.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_action.c @@ -1170,7 +1170,6 @@ mlx5dr_action_create_mult_dest_tbl(struct mlx5dr_domain *dmn, bool ignore_flow_level, u32 flow_source) { - struct mlx5dr_cmd_flow_destination_hw_info tmp_hw_dest; struct mlx5dr_cmd_flow_destination_hw_info *hw_dests; struct mlx5dr_action **ref_actions; struct mlx5dr_action *action; @@ -1249,11 +1248,8 @@ mlx5dr_action_create_mult_dest_tbl(struct mlx5dr_domain *dmn, * one that done in the TX. * So, if one of the ft target is wire, put it at the end of the dest list. */ - if (is_ft_wire && num_dst_ft > 1) { - tmp_hw_dest = hw_dests[last_dest]; - hw_dests[last_dest] = hw_dests[num_of_dests - 1]; - hw_dests[num_of_dests - 1] = tmp_hw_dest; - } + if (is_ft_wire && num_dst_ft > 1) + swap(hw_dests[last_dest], hw_dests[num_of_dests - 1]); action = dr_action_create_generic(DR_ACTION_TYP_FT); if (!action) diff --git a/drivers/net/ethernet/mellanox/mlxsw/reg.h b/drivers/net/ethernet/mellanox/mlxsw/reg.h index 3aae4467e43117..8892654c685f33 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/reg.h +++ b/drivers/net/ethernet/mellanox/mlxsw/reg.h @@ -1954,6 +1954,15 @@ MLXSW_ITEM32(reg, sfmr, irif, 0x14, 0, 16); */ MLXSW_ITEM32(reg, sfmr, cff_mid_base, 0x20, 0, 16); +/* reg_sfmr_nve_flood_prf_id + * FID flooding profile_id for NVE Encap + * Range 0..(max_cap_nve_flood_prf-1) + * Access: RW + * + * Note: Reserved when SwitchX/-2 and Spectrum-1 + */ +MLXSW_ITEM32(reg, sfmr, nve_flood_prf_id, 0x24, 8, 2); + /* reg_sfmr_cff_prf_id * Compressed Fid Flooding profile_id * Range 0..(max_cap_nve_flood_prf-1) diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.h b/drivers/net/ethernet/mellanox/mlxsw/spectrum.h index 61612c41331023..a0c9775fa9552c 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.h +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.h @@ -515,6 +515,8 @@ enum mlxsw_sp_flood_type { MLXSW_SP_FLOOD_TYPE_MC, /* For RSP FIDs in CFF mode. */ MLXSW_SP_FLOOD_TYPE_NOT_UC, + /* For NVE traffic. */ + MLXSW_SP_FLOOD_TYPE_ANY, }; int mlxsw_sp_port_get_stats_raw(struct net_device *dev, int grp, diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_fid.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_fid.c index 4011170862354d..65562ab208b33f 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_fid.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_fid.c @@ -117,6 +117,7 @@ struct mlxsw_sp_fid_ops { enum mlxsw_sp_fid_flood_profile_id { MLXSW_SP_FID_FLOOD_PROFILE_ID_BRIDGE = 1, MLXSW_SP_FID_FLOOD_PROFILE_ID_RSP, + MLXSW_SP_FID_FLOOD_PROFILE_ID_NVE, }; struct mlxsw_sp_fid_flood_profile { @@ -167,11 +168,22 @@ static const int mlxsw_sp_sfgc_not_uc_packet_types[MLXSW_REG_SFGC_TYPE_MAX] = { [MLXSW_REG_SFGC_TYPE_UNREGISTERED_MULTICAST_IPV4] = 1, }; +static const int mlxsw_sp_sfgc_any_packet_types[MLXSW_REG_SFGC_TYPE_MAX] = { + [MLXSW_REG_SFGC_TYPE_UNKNOWN_UNICAST] = 1, + [MLXSW_REG_SFGC_TYPE_BROADCAST] = 1, + [MLXSW_REG_SFGC_TYPE_UNREGISTERED_MULTICAST_NON_IP] = 1, + [MLXSW_REG_SFGC_TYPE_IPV4_LINK_LOCAL] = 1, + [MLXSW_REG_SFGC_TYPE_IPV6_ALL_HOST] = 1, + [MLXSW_REG_SFGC_TYPE_UNREGISTERED_MULTICAST_IPV6] = 1, + [MLXSW_REG_SFGC_TYPE_UNREGISTERED_MULTICAST_IPV4] = 1, +}; + static const int *mlxsw_sp_packet_type_sfgc_types[] = { [MLXSW_SP_FLOOD_TYPE_UC] = mlxsw_sp_sfgc_uc_packet_types, [MLXSW_SP_FLOOD_TYPE_BC] = mlxsw_sp_sfgc_bc_packet_types, [MLXSW_SP_FLOOD_TYPE_MC] = mlxsw_sp_sfgc_mc_packet_types, [MLXSW_SP_FLOOD_TYPE_NOT_UC] = mlxsw_sp_sfgc_not_uc_packet_types, + [MLXSW_SP_FLOOD_TYPE_ANY] = mlxsw_sp_sfgc_any_packet_types, }; struct mlxsw_sp_fid *mlxsw_sp_fid_lookup_by_index(struct mlxsw_sp *mlxsw_sp, @@ -549,6 +561,8 @@ static void mlxsw_sp_fid_fid_pack_cff(char *sfmr_pl, mlxsw_reg_sfmr_cff_mid_base_set(sfmr_pl, pgt_base); mlxsw_reg_sfmr_cff_prf_id_set(sfmr_pl, fid_family->flood_profile->profile_id); + mlxsw_reg_sfmr_nve_flood_prf_id_set(sfmr_pl, + MLXSW_SP_FID_FLOOD_PROFILE_ID_NVE); } static u16 mlxsw_sp_fid_rfid_fid_offset_cff(struct mlxsw_sp *mlxsw_sp, @@ -1310,6 +1324,20 @@ struct mlxsw_sp_fid_flood_profile mlxsw_sp_fid_rsp_flood_profile_cff = { .profile_id = MLXSW_SP_FID_FLOOD_PROFILE_ID_RSP, }; +static const struct mlxsw_sp_flood_table mlxsw_sp_fid_nve_flood_tables_cff[] = { + { + .packet_type = MLXSW_SP_FLOOD_TYPE_ANY, + .table_index = 0, + }, +}; + +static const +struct mlxsw_sp_fid_flood_profile mlxsw_sp_fid_nve_flood_profile_cff = { + .flood_tables = mlxsw_sp_fid_nve_flood_tables_cff, + .nr_flood_tables = ARRAY_SIZE(mlxsw_sp_fid_nve_flood_tables_cff), + .profile_id = MLXSW_SP_FID_FLOOD_PROFILE_ID_NVE, +}; + static bool mlxsw_sp_fid_8021q_compare(const struct mlxsw_sp_fid *fid, const void *arg) { @@ -2411,6 +2439,7 @@ static const struct mlxsw_sp_fid_flood_profile *mlxsw_sp_fid_flood_profiles[] = { &mlxsw_sp_fid_8021d_flood_profile, &mlxsw_sp_fid_rsp_flood_profile_cff, + &mlxsw_sp_fid_nve_flood_profile_cff, }; static int diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/ethernet/microsoft/mana/gdma_main.c index 6367de0c2c2e8f..a6863011d682ea 100644 --- a/drivers/net/ethernet/microsoft/mana/gdma_main.c +++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c @@ -414,8 +414,12 @@ static void mana_gd_process_eq_events(void *arg) old_bits = (eq->head / num_eqe - 1) & GDMA_EQE_OWNER_MASK; /* No more entries */ - if (owner_bits == old_bits) + if (owner_bits == old_bits) { + /* return here without ringing the doorbell */ + if (i == 0) + return; break; + } new_bits = (eq->head / num_eqe) & GDMA_EQE_OWNER_MASK; if (owner_bits != new_bits) { @@ -445,42 +449,29 @@ static int mana_gd_register_irq(struct gdma_queue *queue, struct gdma_dev *gd = queue->gdma_dev; struct gdma_irq_context *gic; struct gdma_context *gc; - struct gdma_resource *r; unsigned int msi_index; unsigned long flags; struct device *dev; int err = 0; gc = gd->gdma_context; - r = &gc->msix_resource; dev = gc->dev; + msi_index = spec->eq.msix_index; - spin_lock_irqsave(&r->lock, flags); - - msi_index = find_first_zero_bit(r->map, r->size); - if (msi_index >= r->size || msi_index >= gc->num_msix_usable) { + if (msi_index >= gc->num_msix_usable) { err = -ENOSPC; - } else { - bitmap_set(r->map, msi_index, 1); - queue->eq.msix_index = msi_index; - } - - spin_unlock_irqrestore(&r->lock, flags); - - if (err) { - dev_err(dev, "Register IRQ err:%d, msi:%u rsize:%u, nMSI:%u", - err, msi_index, r->size, gc->num_msix_usable); + dev_err(dev, "Register IRQ err:%d, msi:%u nMSI:%u", + err, msi_index, gc->num_msix_usable); return err; } + queue->eq.msix_index = msi_index; gic = &gc->irq_contexts[msi_index]; - WARN_ON(gic->handler || gic->arg); - - gic->arg = queue; - - gic->handler = mana_gd_process_eq_events; + spin_lock_irqsave(&gic->lock, flags); + list_add_rcu(&queue->entry, &gic->eq_list); + spin_unlock_irqrestore(&gic->lock, flags); return 0; } @@ -490,12 +481,11 @@ static void mana_gd_deregiser_irq(struct gdma_queue *queue) struct gdma_dev *gd = queue->gdma_dev; struct gdma_irq_context *gic; struct gdma_context *gc; - struct gdma_resource *r; unsigned int msix_index; unsigned long flags; + struct gdma_queue *eq; gc = gd->gdma_context; - r = &gc->msix_resource; /* At most num_online_cpus() + 1 interrupts are used. */ msix_index = queue->eq.msix_index; @@ -503,14 +493,17 @@ static void mana_gd_deregiser_irq(struct gdma_queue *queue) return; gic = &gc->irq_contexts[msix_index]; - gic->handler = NULL; - gic->arg = NULL; - - spin_lock_irqsave(&r->lock, flags); - bitmap_clear(r->map, msix_index, 1); - spin_unlock_irqrestore(&r->lock, flags); + spin_lock_irqsave(&gic->lock, flags); + list_for_each_entry_rcu(eq, &gic->eq_list, entry) { + if (queue == eq) { + list_del_rcu(&eq->entry); + break; + } + } + spin_unlock_irqrestore(&gic->lock, flags); queue->eq.msix_index = INVALID_PCI_MSIX_INDEX; + synchronize_rcu(); } int mana_gd_test_eq(struct gdma_context *gc, struct gdma_queue *eq) @@ -588,6 +581,7 @@ static int mana_gd_create_eq(struct gdma_dev *gd, int err; queue->eq.msix_index = INVALID_PCI_MSIX_INDEX; + queue->id = INVALID_QUEUE_ID; log2_num_entries = ilog2(queue->queue_size / GDMA_EQE_SIZE); @@ -819,6 +813,7 @@ int mana_gd_create_mana_eq(struct gdma_dev *gd, kfree(queue); return err; } +EXPORT_SYMBOL_NS(mana_gd_create_mana_eq, NET_MANA); int mana_gd_create_mana_wq_cq(struct gdma_dev *gd, const struct gdma_queue_spec *spec, @@ -895,6 +890,7 @@ void mana_gd_destroy_queue(struct gdma_context *gc, struct gdma_queue *queue) mana_gd_free_memory(gmi); kfree(queue); } +EXPORT_SYMBOL_NS(mana_gd_destroy_queue, NET_MANA); int mana_gd_verify_vf_version(struct pci_dev *pdev) { @@ -1217,9 +1213,14 @@ int mana_gd_poll_cq(struct gdma_queue *cq, struct gdma_comp *comp, int num_cqe) static irqreturn_t mana_gd_intr(int irq, void *arg) { struct gdma_irq_context *gic = arg; + struct list_head *eq_list = &gic->eq_list; + struct gdma_queue *eq; - if (gic->handler) - gic->handler(gic->arg); + rcu_read_lock(); + list_for_each_entry_rcu(eq, eq_list, entry) { + gic->handler(eq); + } + rcu_read_unlock(); return IRQ_HANDLED; } @@ -1271,8 +1272,9 @@ static int mana_gd_setup_irqs(struct pci_dev *pdev) for (i = 0; i < nvec; i++) { gic = &gc->irq_contexts[i]; - gic->handler = NULL; - gic->arg = NULL; + gic->handler = mana_gd_process_eq_events; + INIT_LIST_HEAD(&gic->eq_list); + spin_lock_init(&gic->lock); if (!i) snprintf(gic->name, MANA_IRQ_NAME_SZ, "mana_hwc@pci:%s", @@ -1295,10 +1297,6 @@ static int mana_gd_setup_irqs(struct pci_dev *pdev) irq_set_affinity_and_hint(irq, cpumask_of(cpu)); } - err = mana_gd_alloc_res_map(nvec, &gc->msix_resource); - if (err) - goto free_irq; - gc->max_num_msix = nvec; gc->num_msix_usable = nvec; @@ -1329,8 +1327,6 @@ static void mana_gd_remove_irqs(struct pci_dev *pdev) if (gc->max_num_msix < 1) return; - mana_gd_free_res_map(&gc->msix_resource); - for (i = 0; i < gc->max_num_msix; i++) { irq = pci_irq_vector(pdev, i); if (irq < 0) diff --git a/drivers/net/ethernet/microsoft/mana/hw_channel.c b/drivers/net/ethernet/microsoft/mana/hw_channel.c index 9d1cd3bfcf6620..2729a2c5acf9cc 100644 --- a/drivers/net/ethernet/microsoft/mana/hw_channel.c +++ b/drivers/net/ethernet/microsoft/mana/hw_channel.c @@ -300,6 +300,7 @@ static int mana_hwc_create_gdma_eq(struct hw_channel_context *hwc, spec.eq.context = ctx; spec.eq.callback = cb; spec.eq.log2_throttle_limit = DEFAULT_LOG2_THROTTLING_FOR_ERROR_EQ; + spec.eq.msix_index = 0; return mana_gd_create_hwc_queue(hwc->gdma_dev, &spec, queue); } diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/ethernet/microsoft/mana/mana_en.c index cb7b9d8ef618ca..59287c6e6cee6f 100644 --- a/drivers/net/ethernet/microsoft/mana/mana_en.c +++ b/drivers/net/ethernet/microsoft/mana/mana_en.c @@ -1244,6 +1244,7 @@ static int mana_create_eq(struct mana_context *ac) spec.eq.log2_throttle_limit = LOG2_EQ_THROTTLE; for (i = 0; i < gc->max_num_queues; i++) { + spec.eq.msix_index = (i + 1) % gc->num_msix_usable; err = mana_gd_create_mana_eq(gd, &spec, &ac->eqs[i].eq); if (err) goto out; diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c index 1ffde555da47c8..70eadc83ca6823 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c @@ -296,62 +296,80 @@ static int stmmac_mtl_setup(struct platform_device *pdev, } /** - * stmmac_dt_phy - parse device-tree driver parameters to allocate PHY resources - * @plat: driver data platform structure - * @np: device tree node - * @dev: device pointer - * Description: - * The mdio bus will be allocated in case of a phy transceiver is on board; - * it will be NULL if the fixed-link is configured. - * If there is the "snps,dwmac-mdio" sub-node the mdio will be allocated - * in any case (for DSA, mdio must be registered even if fixed-link). - * The table below sums the supported configurations: - * ------------------------------- - * snps,phy-addr | Y - * ------------------------------- - * phy-handle | Y - * ------------------------------- - * fixed-link | N - * ------------------------------- - * snps,dwmac-mdio | - * even if | Y - * fixed-link | - * ------------------------------- + * stmmac_of_get_mdio() - Gets the MDIO bus from the devicetree. + * @np: devicetree node * - * It returns 0 in case of success otherwise -ENODEV. + * The MDIO bus will be searched for in the following ways: + * 1. The compatible is "snps,dwc-qos-ethernet-4.10" && a "mdio" named + * child node exists + * 2. A child node with the "snps,dwmac-mdio" compatible is present + * + * Return: The MDIO node if present otherwise NULL */ -static int stmmac_dt_phy(struct plat_stmmacenet_data *plat, - struct device_node *np, struct device *dev) +static struct device_node *stmmac_of_get_mdio(struct device_node *np) { - bool mdio = !of_phy_is_fixed_link(np); static const struct of_device_id need_mdio_ids[] = { { .compatible = "snps,dwc-qos-ethernet-4.10" }, {}, }; + struct device_node *mdio_node = NULL; if (of_match_node(need_mdio_ids, np)) { - plat->mdio_node = of_get_child_by_name(np, "mdio"); + mdio_node = of_get_child_by_name(np, "mdio"); } else { /** * If snps,dwmac-mdio is passed from DT, always register * the MDIO */ - for_each_child_of_node(np, plat->mdio_node) { - if (of_device_is_compatible(plat->mdio_node, + for_each_child_of_node(np, mdio_node) { + if (of_device_is_compatible(mdio_node, "snps,dwmac-mdio")) break; } } - if (plat->mdio_node) { + return mdio_node; +} + +/** + * stmmac_mdio_setup() - Populate platform related MDIO structures. + * @plat: driver data platform structure + * @np: devicetree node + * @dev: device pointer + * + * This searches for MDIO information from the devicetree. + * If an MDIO node is found, it's assigned to plat->mdio_node and + * plat->mdio_bus_data is allocated. + * If no connection can be determined, just plat->mdio_bus_data is allocated + * to indicate a bus should be created and scanned for a phy. + * If it's determined there's no MDIO bus needed, both are left NULL. + * + * This expects that plat->phy_node has already been searched for. + * + * Return: 0 on success, errno otherwise. + */ +static int stmmac_mdio_setup(struct plat_stmmacenet_data *plat, + struct device_node *np, struct device *dev) +{ + bool legacy_mdio; + + plat->mdio_node = stmmac_of_get_mdio(np); + if (plat->mdio_node) dev_dbg(dev, "Found MDIO subnode\n"); - mdio = true; - } - if (mdio) { - plat->mdio_bus_data = - devm_kzalloc(dev, sizeof(struct stmmac_mdio_bus_data), - GFP_KERNEL); + /* Legacy devicetrees allowed for no MDIO bus description and expect + * the bus to be scanned for devices. If there's no phy or fixed-link + * described assume this is the case since there must be something + * connected to the MAC. + */ + legacy_mdio = !of_phy_is_fixed_link(np) && !plat->phy_node; + if (legacy_mdio) + dev_info(dev, "Deprecated MDIO bus assumption used\n"); + + if (plat->mdio_node || legacy_mdio) { + plat->mdio_bus_data = devm_kzalloc(dev, + sizeof(*plat->mdio_bus_data), + GFP_KERNEL); if (!plat->mdio_bus_data) return -ENOMEM; @@ -471,8 +489,7 @@ stmmac_probe_config_dt(struct platform_device *pdev, u8 *mac) if (of_property_read_u32(np, "snps,phy-addr", &plat->phy_addr) == 0) dev_warn(&pdev->dev, "snps,phy-addr property is deprecated\n"); - /* To Configure PHY by using all device-tree supported properties */ - rc = stmmac_dt_phy(plat, np, &pdev->dev); + rc = stmmac_mdio_setup(plat, np, &pdev->dev); if (rc) return ERR_PTR(rc); diff --git a/drivers/net/hyperv/rndis_filter.c b/drivers/net/hyperv/rndis_filter.c index af95947a87c552..ecc2128ca9b72a 100644 --- a/drivers/net/hyperv/rndis_filter.c +++ b/drivers/net/hyperv/rndis_filter.c @@ -21,7 +21,6 @@ #include #include #include -#include #include "hyperv_net.h" #include "netvsc_trace.h" diff --git a/drivers/net/phy/Kconfig b/drivers/net/phy/Kconfig index 098d17d2d3f75d..2e4667bf9ff520 100644 --- a/drivers/net/phy/Kconfig +++ b/drivers/net/phy/Kconfig @@ -60,6 +60,14 @@ config FIXED_PHY Currently tested with mpc866ads and mpc8349e-mitx. +config RUST_PHYLIB_ABSTRACTIONS + bool "Rust PHYLIB abstractions support" + depends on RUST + depends on PHYLIB=y + help + Adds support needed for PHY drivers written in Rust. It provides + a wrapper around the C phylib core. + config SFP tristate "SFP cage support" depends on I2C && PHYLINK @@ -104,6 +112,14 @@ config AX88796B_PHY Currently supports the Asix Electronics PHY found in the X-Surf 100 AX88796B package. +config AX88796B_RUST_PHY + bool "Rust reference driver for Asix PHYs" + depends on RUST_PHYLIB_ABSTRACTIONS && AX88796B_PHY + help + Uses the Rust reference driver for Asix PHYs (ax88796b_rust.ko). + The features are equivalent. It supports the Asix Electronics PHY + found in the X-Surf 100 AX88796B package. + config BROADCOM_PHY tristate "Broadcom 54XX PHYs" select BCM_NET_PHYLIB diff --git a/drivers/net/phy/Makefile b/drivers/net/phy/Makefile index 4ace41095ee2b3..e35ea69d9cb4a8 100644 --- a/drivers/net/phy/Makefile +++ b/drivers/net/phy/Makefile @@ -37,7 +37,11 @@ obj-$(CONFIG_ADIN1100_PHY) += adin1100.o obj-$(CONFIG_AMD_PHY) += amd.o obj-$(CONFIG_AQUANTIA_PHY) += aquantia/ obj-$(CONFIG_AT803X_PHY) += at803x.o -obj-$(CONFIG_AX88796B_PHY) += ax88796b.o +ifdef CONFIG_AX88796B_RUST_PHY + obj-$(CONFIG_AX88796B_PHY) += ax88796b_rust.o +else + obj-$(CONFIG_AX88796B_PHY) += ax88796b.o +endif obj-$(CONFIG_BCM54140_PHY) += bcm54140.o obj-$(CONFIG_BCM63XX_PHY) += bcm63xx.o obj-$(CONFIG_BCM7XXX_PHY) += bcm7xxx.o diff --git a/drivers/net/phy/at803x.c b/drivers/net/phy/at803x.c index b9d3a26cf6dcb8..a7d28848ee9331 100644 --- a/drivers/net/phy/at803x.c +++ b/drivers/net/phy/at803x.c @@ -301,6 +301,11 @@ static struct at803x_hw_stat qca83xx_hw_stats[] = { { "eee_wake_errors", 0x16, GENMASK(15, 0), MMD}, }; +struct at803x_ss_mask { + u16 speed_mask; + u8 speed_shift; +}; + struct at803x_priv { int flags; u16 clk_25m_reg; @@ -921,7 +926,8 @@ static void at803x_link_change_notify(struct phy_device *phydev) } } -static int at803x_read_specific_status(struct phy_device *phydev) +static int at803x_read_specific_status(struct phy_device *phydev, + struct at803x_ss_mask ss_mask) { int ss; @@ -940,11 +946,8 @@ static int at803x_read_specific_status(struct phy_device *phydev) if (sfc < 0) return sfc; - /* qca8081 takes the different bits for speed value from at803x */ - if (phydev->drv->phy_id == QCA8081_PHY_ID) - speed = FIELD_GET(QCA808X_SS_SPEED_MASK, ss); - else - speed = FIELD_GET(AT803X_SS_SPEED_MASK, ss); + speed = ss & ss_mask.speed_mask; + speed >>= ss_mask.speed_shift; switch (speed) { case AT803X_SS_SPEED_10: @@ -989,6 +992,7 @@ static int at803x_read_specific_status(struct phy_device *phydev) static int at803x_read_status(struct phy_device *phydev) { struct at803x_priv *priv = phydev->priv; + struct at803x_ss_mask ss_mask = { 0 }; int err, old_link = phydev->link; if (priv->is_1000basex) @@ -1012,7 +1016,9 @@ static int at803x_read_status(struct phy_device *phydev) if (err < 0) return err; - err = at803x_read_specific_status(phydev); + ss_mask.speed_mask = AT803X_SS_SPEED_MASK; + ss_mask.speed_shift = __bf_shf(AT803X_SS_SPEED_MASK); + err = at803x_read_specific_status(phydev, ss_mask); if (err < 0) return err; @@ -1045,9 +1051,8 @@ static int at803x_config_mdix(struct phy_device *phydev, u8 ctrl) FIELD_PREP(AT803X_SFC_MDI_CROSSOVER_MODE_M, val)); } -static int at803x_config_aneg(struct phy_device *phydev) +static int at803x_prepare_config_aneg(struct phy_device *phydev) { - struct at803x_priv *priv = phydev->priv; int ret; ret = at803x_config_mdix(phydev, phydev->mdix_ctrl); @@ -1064,33 +1069,22 @@ static int at803x_config_aneg(struct phy_device *phydev) return ret; } - if (priv->is_1000basex) - return genphy_c37_config_aneg(phydev); - - /* Do not restart auto-negotiation by setting ret to 0 defautly, - * when calling __genphy_config_aneg later. - */ - ret = 0; - - if (phydev->drv->phy_id == QCA8081_PHY_ID) { - int phy_ctrl = 0; + return 0; +} - /* The reg MII_BMCR also needs to be configured for force mode, the - * genphy_config_aneg is also needed. - */ - if (phydev->autoneg == AUTONEG_DISABLE) - genphy_c45_pma_setup_forced(phydev); +static int at803x_config_aneg(struct phy_device *phydev) +{ + struct at803x_priv *priv = phydev->priv; + int ret; - if (linkmode_test_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, phydev->advertising)) - phy_ctrl = MDIO_AN_10GBT_CTRL_ADV2_5G; + ret = at803x_prepare_config_aneg(phydev); + if (ret) + return ret; - ret = phy_modify_mmd_changed(phydev, MDIO_MMD_AN, MDIO_AN_10GBT_CTRL, - MDIO_AN_10GBT_CTRL_ADV2_5G, phy_ctrl); - if (ret < 0) - return ret; - } + if (priv->is_1000basex) + return genphy_c37_config_aneg(phydev); - return __genphy_config_aneg(phydev, ret); + return genphy_config_aneg(phydev); } static int at803x_get_downshift(struct phy_device *phydev, u8 *d) @@ -1881,6 +1875,7 @@ static int qca808x_config_init(struct phy_device *phydev) static int qca808x_read_status(struct phy_device *phydev) { + struct at803x_ss_mask ss_mask = { 0 }; int ret; ret = phy_read_mmd(phydev, MDIO_MMD_AN, MDIO_AN_10GBT_STAT); @@ -1894,7 +1889,10 @@ static int qca808x_read_status(struct phy_device *phydev) if (ret) return ret; - ret = at803x_read_specific_status(phydev); + /* qca8081 takes the different bits for speed value from at803x */ + ss_mask.speed_mask = QCA808X_SS_SPEED_MASK; + ss_mask.speed_shift = __bf_shf(QCA808X_SS_SPEED_MASK); + ret = at803x_read_specific_status(phydev, ss_mask); if (ret < 0) return ret; @@ -2118,6 +2116,32 @@ static int qca808x_get_features(struct phy_device *phydev) return 0; } +static int qca808x_config_aneg(struct phy_device *phydev) +{ + int phy_ctrl = 0; + int ret; + + ret = at803x_prepare_config_aneg(phydev); + if (ret) + return ret; + + /* The reg MII_BMCR also needs to be configured for force mode, the + * genphy_config_aneg is also needed. + */ + if (phydev->autoneg == AUTONEG_DISABLE) + genphy_c45_pma_setup_forced(phydev); + + if (linkmode_test_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, phydev->advertising)) + phy_ctrl = MDIO_AN_10GBT_CTRL_ADV2_5G; + + ret = phy_modify_mmd_changed(phydev, MDIO_MMD_AN, MDIO_AN_10GBT_CTRL, + MDIO_AN_10GBT_CTRL_ADV2_5G, phy_ctrl); + if (ret < 0) + return ret; + + return __genphy_config_aneg(phydev, ret); +} + static void qca808x_link_change_notify(struct phy_device *phydev) { /* Assert interface sgmii fifo on link down, deassert it on link up, @@ -2295,7 +2319,7 @@ static struct phy_driver at803x_driver[] = { .set_wol = at803x_set_wol, .get_wol = at803x_get_wol, .get_features = qca808x_get_features, - .config_aneg = at803x_config_aneg, + .config_aneg = qca808x_config_aneg, .suspend = genphy_suspend, .resume = genphy_resume, .read_status = qca808x_read_status, diff --git a/drivers/net/phy/ax88796b_rust.rs b/drivers/net/phy/ax88796b_rust.rs new file mode 100644 index 00000000000000..5c92572962dce7 --- /dev/null +++ b/drivers/net/phy/ax88796b_rust.rs @@ -0,0 +1,135 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2023 FUJITA Tomonori + +//! Rust Asix PHYs driver +//! +//! C version of this driver: [`drivers/net/phy/ax88796b.c`](./ax88796b.c) +use kernel::{ + c_str, + net::phy::{self, DeviceId, Driver}, + prelude::*, + uapi, +}; + +kernel::module_phy_driver! { + drivers: [PhyAX88772A, PhyAX88772C, PhyAX88796B], + device_table: [ + DeviceId::new_with_driver::(), + DeviceId::new_with_driver::(), + DeviceId::new_with_driver::() + ], + name: "rust_asix_phy", + author: "FUJITA Tomonori ", + description: "Rust Asix PHYs driver", + license: "GPL", +} + +const MII_BMCR: u16 = uapi::MII_BMCR as u16; +const BMCR_SPEED100: u16 = uapi::BMCR_SPEED100 as u16; +const BMCR_FULLDPLX: u16 = uapi::BMCR_FULLDPLX as u16; + +// Performs a software PHY reset using the standard +// BMCR_RESET bit and poll for the reset bit to be cleared. +// Toggle BMCR_RESET bit off to accommodate broken AX8796B PHY implementation +// such as used on the Individual Computers' X-Surf 100 Zorro card. +fn asix_soft_reset(dev: &mut phy::Device) -> Result { + dev.write(uapi::MII_BMCR as u16, 0)?; + dev.genphy_soft_reset() +} + +struct PhyAX88772A; + +#[vtable] +impl Driver for PhyAX88772A { + const FLAGS: u32 = phy::flags::IS_INTERNAL; + const NAME: &'static CStr = c_str!("Asix Electronics AX88772A"); + const PHY_DEVICE_ID: DeviceId = DeviceId::new_with_exact_mask(0x003b1861); + + // AX88772A is not working properly with some old switches (NETGEAR EN 108TP): + // after autoneg is done and the link status is reported as active, the MII_LPA + // register is 0. This issue is not reproducible on AX88772C. + fn read_status(dev: &mut phy::Device) -> Result { + dev.genphy_update_link()?; + if !dev.is_link_up() { + return Ok(0); + } + // If MII_LPA is 0, phy_resolve_aneg_linkmode() will fail to resolve + // linkmode so use MII_BMCR as default values. + let ret = dev.read(MII_BMCR)?; + + if ret & BMCR_SPEED100 != 0 { + dev.set_speed(uapi::SPEED_100); + } else { + dev.set_speed(uapi::SPEED_10); + } + + let duplex = if ret & BMCR_FULLDPLX != 0 { + phy::DuplexMode::Full + } else { + phy::DuplexMode::Half + }; + dev.set_duplex(duplex); + + dev.genphy_read_lpa()?; + + if dev.is_autoneg_enabled() && dev.is_autoneg_completed() { + dev.resolve_aneg_linkmode(); + } + + Ok(0) + } + + fn suspend(dev: &mut phy::Device) -> Result { + dev.genphy_suspend() + } + + fn resume(dev: &mut phy::Device) -> Result { + dev.genphy_resume() + } + + fn soft_reset(dev: &mut phy::Device) -> Result { + asix_soft_reset(dev) + } + + fn link_change_notify(dev: &mut phy::Device) { + // Reset PHY, otherwise MII_LPA will provide outdated information. + // This issue is reproducible only with some link partner PHYs. + if dev.state() == phy::DeviceState::NoLink { + let _ = dev.init_hw(); + let _ = dev.start_aneg(); + } + } +} + +struct PhyAX88772C; + +#[vtable] +impl Driver for PhyAX88772C { + const FLAGS: u32 = phy::flags::IS_INTERNAL; + const NAME: &'static CStr = c_str!("Asix Electronics AX88772C"); + const PHY_DEVICE_ID: DeviceId = DeviceId::new_with_exact_mask(0x003b1881); + + fn suspend(dev: &mut phy::Device) -> Result { + dev.genphy_suspend() + } + + fn resume(dev: &mut phy::Device) -> Result { + dev.genphy_resume() + } + + fn soft_reset(dev: &mut phy::Device) -> Result { + asix_soft_reset(dev) + } +} + +struct PhyAX88796B; + +#[vtable] +impl Driver for PhyAX88796B { + const NAME: &'static CStr = c_str!("Asix Electronics AX88796B"); + const PHY_DEVICE_ID: DeviceId = DeviceId::new_with_model_mask(0x003b1841); + + fn soft_reset(dev: &mut phy::Device) -> Result { + asix_soft_reset(dev) + } +} diff --git a/drivers/net/phy/bcm54140.c b/drivers/net/phy/bcm54140.c index d43076592f81d4..2eea3d09b1e6f8 100644 --- a/drivers/net/phy/bcm54140.c +++ b/drivers/net/phy/bcm54140.c @@ -128,6 +128,10 @@ #define BCM54140_DEFAULT_DOWNSHIFT 5 #define BCM54140_MAX_DOWNSHIFT 9 +enum bcm54140_global_phy { + BCM54140_BASE_ADDR = 0, +}; + struct bcm54140_priv { int port; int base_addr; @@ -429,11 +433,13 @@ static int bcm54140_base_read_rdb(struct phy_device *phydev, u16 rdb) int ret; phy_lock_mdio_bus(phydev); - ret = __phy_package_write(phydev, MII_BCM54XX_RDB_ADDR, rdb); + ret = __phy_package_write(phydev, BCM54140_BASE_ADDR, + MII_BCM54XX_RDB_ADDR, rdb); if (ret < 0) goto out; - ret = __phy_package_read(phydev, MII_BCM54XX_RDB_DATA); + ret = __phy_package_read(phydev, BCM54140_BASE_ADDR, + MII_BCM54XX_RDB_DATA); out: phy_unlock_mdio_bus(phydev); @@ -446,11 +452,13 @@ static int bcm54140_base_write_rdb(struct phy_device *phydev, int ret; phy_lock_mdio_bus(phydev); - ret = __phy_package_write(phydev, MII_BCM54XX_RDB_ADDR, rdb); + ret = __phy_package_write(phydev, BCM54140_BASE_ADDR, + MII_BCM54XX_RDB_ADDR, rdb); if (ret < 0) goto out; - ret = __phy_package_write(phydev, MII_BCM54XX_RDB_DATA, val); + ret = __phy_package_write(phydev, BCM54140_BASE_ADDR, + MII_BCM54XX_RDB_DATA, val); out: phy_unlock_mdio_bus(phydev); diff --git a/drivers/net/phy/mscc/mscc.h b/drivers/net/phy/mscc/mscc.h index 7a962050a4d45e..6a3d8a754eb8de 100644 --- a/drivers/net/phy/mscc/mscc.h +++ b/drivers/net/phy/mscc/mscc.h @@ -416,6 +416,11 @@ struct vsc8531_private { * gpio_lock: used for PHC operations. Common for all PHYs as the load/save GPIO * is shared. */ + +enum vsc85xx_global_phy { + VSC88XX_BASE_ADDR = 0, +}; + struct vsc85xx_shared_private { struct mutex gpio_lock; }; diff --git a/drivers/net/phy/mscc/mscc_main.c b/drivers/net/phy/mscc/mscc_main.c index 4171f01d34e579..6f74ce0ab1aad8 100644 --- a/drivers/net/phy/mscc/mscc_main.c +++ b/drivers/net/phy/mscc/mscc_main.c @@ -711,7 +711,7 @@ int phy_base_write(struct phy_device *phydev, u32 regnum, u16 val) dump_stack(); } - return __phy_package_write(phydev, regnum, val); + return __phy_package_write(phydev, VSC88XX_BASE_ADDR, regnum, val); } /* phydev->bus->mdio_lock should be locked when using this function */ @@ -722,7 +722,7 @@ int phy_base_read(struct phy_device *phydev, u32 regnum) dump_stack(); } - return __phy_package_read(phydev, regnum); + return __phy_package_read(phydev, VSC88XX_BASE_ADDR, regnum); } u32 vsc85xx_csr_read(struct phy_device *phydev, diff --git a/drivers/net/phy/phy-core.c b/drivers/net/phy/phy-core.c index 966c93cbe61641..15f349e5995a68 100644 --- a/drivers/net/phy/phy-core.c +++ b/drivers/net/phy/phy-core.c @@ -540,6 +540,28 @@ static void mmd_phy_indirect(struct mii_bus *bus, int phy_addr, int devad, devad | MII_MMD_CTRL_NOINCR); } +static int mmd_phy_read(struct mii_bus *bus, int phy_addr, bool is_c45, + int devad, u32 regnum) +{ + if (is_c45) + return __mdiobus_c45_read(bus, phy_addr, devad, regnum); + + mmd_phy_indirect(bus, phy_addr, devad, regnum); + /* Read the content of the MMD's selected register */ + return __mdiobus_read(bus, phy_addr, MII_MMD_DATA); +} + +static int mmd_phy_write(struct mii_bus *bus, int phy_addr, bool is_c45, + int devad, u32 regnum, u16 val) +{ + if (is_c45) + return __mdiobus_c45_write(bus, phy_addr, devad, regnum, val); + + mmd_phy_indirect(bus, phy_addr, devad, regnum); + /* Write the data into MMD's selected register */ + return __mdiobus_write(bus, phy_addr, MII_MMD_DATA, val); +} + /** * __phy_read_mmd - Convenience function for reading a register * from an MMD on a given PHY. @@ -551,26 +573,14 @@ static void mmd_phy_indirect(struct mii_bus *bus, int phy_addr, int devad, */ int __phy_read_mmd(struct phy_device *phydev, int devad, u32 regnum) { - int val; - if (regnum > (u16)~0 || devad > 32) return -EINVAL; - if (phydev->drv && phydev->drv->read_mmd) { - val = phydev->drv->read_mmd(phydev, devad, regnum); - } else if (phydev->is_c45) { - val = __mdiobus_c45_read(phydev->mdio.bus, phydev->mdio.addr, - devad, regnum); - } else { - struct mii_bus *bus = phydev->mdio.bus; - int phy_addr = phydev->mdio.addr; - - mmd_phy_indirect(bus, phy_addr, devad, regnum); + if (phydev->drv && phydev->drv->read_mmd) + return phydev->drv->read_mmd(phydev, devad, regnum); - /* Read the content of the MMD's selected register */ - val = __mdiobus_read(bus, phy_addr, MII_MMD_DATA); - } - return val; + return mmd_phy_read(phydev->mdio.bus, phydev->mdio.addr, + phydev->is_c45, devad, regnum); } EXPORT_SYMBOL(__phy_read_mmd); @@ -607,28 +617,14 @@ EXPORT_SYMBOL(phy_read_mmd); */ int __phy_write_mmd(struct phy_device *phydev, int devad, u32 regnum, u16 val) { - int ret; - if (regnum > (u16)~0 || devad > 32) return -EINVAL; - if (phydev->drv && phydev->drv->write_mmd) { - ret = phydev->drv->write_mmd(phydev, devad, regnum, val); - } else if (phydev->is_c45) { - ret = __mdiobus_c45_write(phydev->mdio.bus, phydev->mdio.addr, - devad, regnum, val); - } else { - struct mii_bus *bus = phydev->mdio.bus; - int phy_addr = phydev->mdio.addr; - - mmd_phy_indirect(bus, phy_addr, devad, regnum); + if (phydev->drv && phydev->drv->write_mmd) + return phydev->drv->write_mmd(phydev, devad, regnum, val); - /* Write the data into MMD's selected register */ - __mdiobus_write(bus, phy_addr, MII_MMD_DATA, val); - - ret = 0; - } - return ret; + return mmd_phy_write(phydev->mdio.bus, phydev->mdio.addr, + phydev->is_c45, devad, regnum, val); } EXPORT_SYMBOL(__phy_write_mmd); @@ -654,6 +650,146 @@ int phy_write_mmd(struct phy_device *phydev, int devad, u32 regnum, u16 val) } EXPORT_SYMBOL(phy_write_mmd); +/** + * __phy_package_read_mmd - read MMD reg relative to PHY package base addr + * @phydev: The phy_device struct + * @addr_offset: The offset to be added to PHY package base_addr + * @devad: The MMD to read from + * @regnum: The register on the MMD to read + * + * Convenience helper for reading a register of an MMD on a given PHY + * using the PHY package base address. The base address is added to + * the addr_offset value. + * + * Same calling rules as for __phy_read(); + * + * NOTE: It's assumed that the entire PHY package is either C22 or C45. + */ +int __phy_package_read_mmd(struct phy_device *phydev, + unsigned int addr_offset, int devad, + u32 regnum) +{ + int addr = phy_package_address(phydev, addr_offset); + + if (addr < 0) + return addr; + + if (regnum > (u16)~0 || devad > 32) + return -EINVAL; + + return mmd_phy_read(phydev->mdio.bus, addr, phydev->is_c45, devad, + regnum); +} +EXPORT_SYMBOL(__phy_package_read_mmd); + +/** + * phy_package_read_mmd - read MMD reg relative to PHY package base addr + * @phydev: The phy_device struct + * @addr_offset: The offset to be added to PHY package base_addr + * @devad: The MMD to read from + * @regnum: The register on the MMD to read + * + * Convenience helper for reading a register of an MMD on a given PHY + * using the PHY package base address. The base address is added to + * the addr_offset value. + * + * Same calling rules as for phy_read(); + * + * NOTE: It's assumed that the entire PHY package is either C22 or C45. + */ +int phy_package_read_mmd(struct phy_device *phydev, + unsigned int addr_offset, int devad, + u32 regnum) +{ + int addr = phy_package_address(phydev, addr_offset); + int val; + + if (addr < 0) + return addr; + + if (regnum > (u16)~0 || devad > 32) + return -EINVAL; + + phy_lock_mdio_bus(phydev); + val = mmd_phy_read(phydev->mdio.bus, addr, phydev->is_c45, devad, + regnum); + phy_unlock_mdio_bus(phydev); + + return val; +} +EXPORT_SYMBOL(phy_package_read_mmd); + +/** + * __phy_package_write_mmd - write MMD reg relative to PHY package base addr + * @phydev: The phy_device struct + * @addr_offset: The offset to be added to PHY package base_addr + * @devad: The MMD to write to + * @regnum: The register on the MMD to write + * @val: value to write to @regnum + * + * Convenience helper for writing a register of an MMD on a given PHY + * using the PHY package base address. The base address is added to + * the addr_offset value. + * + * Same calling rules as for __phy_write(); + * + * NOTE: It's assumed that the entire PHY package is either C22 or C45. + */ +int __phy_package_write_mmd(struct phy_device *phydev, + unsigned int addr_offset, int devad, + u32 regnum, u16 val) +{ + int addr = phy_package_address(phydev, addr_offset); + + if (addr < 0) + return addr; + + if (regnum > (u16)~0 || devad > 32) + return -EINVAL; + + return mmd_phy_write(phydev->mdio.bus, addr, phydev->is_c45, devad, + regnum, val); +} +EXPORT_SYMBOL(__phy_package_write_mmd); + +/** + * phy_package_write_mmd - write MMD reg relative to PHY package base addr + * @phydev: The phy_device struct + * @addr_offset: The offset to be added to PHY package base_addr + * @devad: The MMD to write to + * @regnum: The register on the MMD to write + * @val: value to write to @regnum + * + * Convenience helper for writing a register of an MMD on a given PHY + * using the PHY package base address. The base address is added to + * the addr_offset value. + * + * Same calling rules as for phy_write(); + * + * NOTE: It's assumed that the entire PHY package is either C22 or C45. + */ +int phy_package_write_mmd(struct phy_device *phydev, + unsigned int addr_offset, int devad, + u32 regnum, u16 val) +{ + int addr = phy_package_address(phydev, addr_offset); + int ret; + + if (addr < 0) + return addr; + + if (regnum > (u16)~0 || devad > 32) + return -EINVAL; + + phy_lock_mdio_bus(phydev); + ret = mmd_phy_write(phydev->mdio.bus, addr, phydev->is_c45, devad, + regnum, val); + phy_unlock_mdio_bus(phydev); + + return ret; +} +EXPORT_SYMBOL(phy_package_write_mmd); + /** * phy_modify_changed - Function for modifying a PHY register * @phydev: the phy_device struct diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c index d8e9335d415ca5..0c52a9eff188da 100644 --- a/drivers/net/phy/phy_device.c +++ b/drivers/net/phy/phy_device.c @@ -1651,20 +1651,22 @@ EXPORT_SYMBOL_GPL(phy_driver_is_genphy_10g); /** * phy_package_join - join a common PHY group * @phydev: target phy_device struct - * @addr: cookie and PHY address for global register access + * @base_addr: cookie and base PHY address of PHY package for offset + * calculation of global register access * @priv_size: if non-zero allocate this amount of bytes for private data * * This joins a PHY group and provides a shared storage for all phydevs in * this group. This is intended to be used for packages which contain * more than one PHY, for example a quad PHY transceiver. * - * The addr parameter serves as a cookie which has to have the same value - * for all members of one group and as a PHY address to access generic - * registers of a PHY package. Usually, one of the PHY addresses of the - * different PHYs in the package provides access to these global registers. + * The base_addr parameter serves as cookie which has to have the same values + * for all members of one group and as the base PHY address of the PHY package + * for offset calculation to access generic registers of a PHY package. + * Usually, one of the PHY addresses of the different PHYs in the package + * provides access to these global registers. * The address which is given here, will be used in the phy_package_read() - * and phy_package_write() convenience functions. If your PHY doesn't have - * global registers you can just pick any of the PHY addresses. + * and phy_package_write() convenience functions as base and added to the + * passed offset in those functions. * * This will set the shared pointer of the phydev to the shared storage. * If this is the first call for a this cookie the shared storage will be @@ -1674,17 +1676,17 @@ EXPORT_SYMBOL_GPL(phy_driver_is_genphy_10g); * Returns < 1 on error, 0 on success. Esp. calling phy_package_join() * with the same cookie but a different priv_size is an error. */ -int phy_package_join(struct phy_device *phydev, int addr, size_t priv_size) +int phy_package_join(struct phy_device *phydev, int base_addr, size_t priv_size) { struct mii_bus *bus = phydev->mdio.bus; struct phy_package_shared *shared; int ret; - if (addr < 0 || addr >= PHY_MAX_ADDR) + if (base_addr < 0 || base_addr >= PHY_MAX_ADDR) return -EINVAL; mutex_lock(&bus->shared_lock); - shared = bus->shared[addr]; + shared = bus->shared[base_addr]; if (!shared) { ret = -ENOMEM; shared = kzalloc(sizeof(*shared), GFP_KERNEL); @@ -1696,9 +1698,9 @@ int phy_package_join(struct phy_device *phydev, int addr, size_t priv_size) goto err_free; shared->priv_size = priv_size; } - shared->addr = addr; + shared->base_addr = base_addr; refcount_set(&shared->refcnt, 1); - bus->shared[addr] = shared; + bus->shared[base_addr] = shared; } else { ret = -EINVAL; if (priv_size && priv_size != shared->priv_size) @@ -1736,7 +1738,7 @@ void phy_package_leave(struct phy_device *phydev) return; if (refcount_dec_and_mutex_lock(&shared->refcnt, &bus->shared_lock)) { - bus->shared[shared->addr] = NULL; + bus->shared[shared->base_addr] = NULL; mutex_unlock(&bus->shared_lock); kfree(shared->priv); kfree(shared); @@ -1755,7 +1757,8 @@ static void devm_phy_package_leave(struct device *dev, void *res) * devm_phy_package_join - resource managed phy_package_join() * @dev: device that is registering this PHY package * @phydev: target phy_device struct - * @addr: cookie and PHY address for global register access + * @base_addr: cookie and base PHY address of PHY package for offset + * calculation of global register access * @priv_size: if non-zero allocate this amount of bytes for private data * * Managed phy_package_join(). Shared storage fetched by this function, @@ -1763,7 +1766,7 @@ static void devm_phy_package_leave(struct device *dev, void *res) * phy_package_join() for more information. */ int devm_phy_package_join(struct device *dev, struct phy_device *phydev, - int addr, size_t priv_size) + int base_addr, size_t priv_size) { struct phy_device **ptr; int ret; @@ -1773,7 +1776,7 @@ int devm_phy_package_join(struct device *dev, struct phy_device *phydev, if (!ptr) return -ENOMEM; - ret = phy_package_join(phydev, addr, priv_size); + ret = phy_package_join(phydev, base_addr, priv_size); if (!ret) { *ptr = phydev; diff --git a/drivers/net/phy/phylink.c b/drivers/net/phy/phylink.c index 48d3bd3e9fc744..298dfd6982a504 100644 --- a/drivers/net/phy/phylink.c +++ b/drivers/net/phy/phylink.c @@ -883,6 +883,7 @@ static int phylink_parse_mode(struct phylink *pl, { struct fwnode_handle *dn; const char *managed; + unsigned long caps; dn = fwnode_get_named_child_node(fwnode, "fixed-link"); if (dn || fwnode_property_present(fwnode, "fixed-link")) @@ -915,80 +916,18 @@ static int phylink_parse_mode(struct phylink *pl, case PHY_INTERFACE_MODE_RGMII_RXID: case PHY_INTERFACE_MODE_RGMII_TXID: case PHY_INTERFACE_MODE_RTBI: - phylink_set(pl->supported, 10baseT_Half); - phylink_set(pl->supported, 10baseT_Full); - phylink_set(pl->supported, 100baseT_Half); - phylink_set(pl->supported, 100baseT_Full); - phylink_set(pl->supported, 1000baseT_Half); - phylink_set(pl->supported, 1000baseT_Full); - break; - case PHY_INTERFACE_MODE_1000BASEX: - phylink_set(pl->supported, 1000baseX_Full); - break; - case PHY_INTERFACE_MODE_2500BASEX: - phylink_set(pl->supported, 2500baseX_Full); - break; - case PHY_INTERFACE_MODE_5GBASER: - phylink_set(pl->supported, 5000baseT_Full); - break; - case PHY_INTERFACE_MODE_25GBASER: - phylink_set(pl->supported, 25000baseCR_Full); - phylink_set(pl->supported, 25000baseKR_Full); - phylink_set(pl->supported, 25000baseSR_Full); - fallthrough; case PHY_INTERFACE_MODE_USXGMII: case PHY_INTERFACE_MODE_10GKR: case PHY_INTERFACE_MODE_10GBASER: - phylink_set(pl->supported, 10baseT_Half); - phylink_set(pl->supported, 10baseT_Full); - phylink_set(pl->supported, 100baseT_Half); - phylink_set(pl->supported, 100baseT_Full); - phylink_set(pl->supported, 1000baseT_Half); - phylink_set(pl->supported, 1000baseT_Full); - phylink_set(pl->supported, 1000baseX_Full); - phylink_set(pl->supported, 1000baseKX_Full); - phylink_set(pl->supported, 2500baseT_Full); - phylink_set(pl->supported, 2500baseX_Full); - phylink_set(pl->supported, 5000baseT_Full); - phylink_set(pl->supported, 10000baseT_Full); - phylink_set(pl->supported, 10000baseKR_Full); - phylink_set(pl->supported, 10000baseKX4_Full); - phylink_set(pl->supported, 10000baseCR_Full); - phylink_set(pl->supported, 10000baseSR_Full); - phylink_set(pl->supported, 10000baseLR_Full); - phylink_set(pl->supported, 10000baseLRM_Full); - phylink_set(pl->supported, 10000baseER_Full); - break; - case PHY_INTERFACE_MODE_XLGMII: - phylink_set(pl->supported, 25000baseCR_Full); - phylink_set(pl->supported, 25000baseKR_Full); - phylink_set(pl->supported, 25000baseSR_Full); - phylink_set(pl->supported, 40000baseKR4_Full); - phylink_set(pl->supported, 40000baseCR4_Full); - phylink_set(pl->supported, 40000baseSR4_Full); - phylink_set(pl->supported, 40000baseLR4_Full); - phylink_set(pl->supported, 50000baseCR2_Full); - phylink_set(pl->supported, 50000baseKR2_Full); - phylink_set(pl->supported, 50000baseSR2_Full); - phylink_set(pl->supported, 50000baseKR_Full); - phylink_set(pl->supported, 50000baseSR_Full); - phylink_set(pl->supported, 50000baseCR_Full); - phylink_set(pl->supported, 50000baseLR_ER_FR_Full); - phylink_set(pl->supported, 50000baseDR_Full); - phylink_set(pl->supported, 100000baseKR4_Full); - phylink_set(pl->supported, 100000baseSR4_Full); - phylink_set(pl->supported, 100000baseCR4_Full); - phylink_set(pl->supported, 100000baseLR4_ER4_Full); - phylink_set(pl->supported, 100000baseKR2_Full); - phylink_set(pl->supported, 100000baseSR2_Full); - phylink_set(pl->supported, 100000baseCR2_Full); - phylink_set(pl->supported, 100000baseLR2_ER2_FR2_Full); - phylink_set(pl->supported, 100000baseDR2_Full); + caps = ~(MAC_SYM_PAUSE | MAC_ASYM_PAUSE); + caps = phylink_get_capabilities(pl->link_config.interface, caps, + RATE_MATCH_NONE); + phylink_caps_to_linkmodes(pl->supported, caps); break; default: diff --git a/drivers/net/veth.c b/drivers/net/veth.c index 977861c46b1fe1..578e36ea1589c1 100644 --- a/drivers/net/veth.c +++ b/drivers/net/veth.c @@ -1723,6 +1723,24 @@ static int veth_xdp_rx_hash(const struct xdp_md *ctx, u32 *hash, return 0; } +static int veth_xdp_rx_vlan_tag(const struct xdp_md *ctx, __be16 *vlan_proto, + u16 *vlan_tci) +{ + const struct veth_xdp_buff *_ctx = (void *)ctx; + const struct sk_buff *skb = _ctx->skb; + int err; + + if (!skb) + return -ENODATA; + + err = __vlan_hwaccel_get_tag(skb, vlan_tci); + if (err) + return err; + + *vlan_proto = skb->vlan_proto; + return err; +} + static const struct net_device_ops veth_netdev_ops = { .ndo_init = veth_dev_init, .ndo_open = veth_open, @@ -1747,6 +1765,7 @@ static const struct net_device_ops veth_netdev_ops = { static const struct xdp_metadata_ops veth_xdp_metadata_ops = { .xmo_rx_timestamp = veth_xdp_rx_timestamp, .xmo_rx_hash = veth_xdp_rx_hash, + .xmo_rx_vlan_tag = veth_xdp_rx_vlan_tag, }; #define VETH_FEATURES (NETIF_F_SG | NETIF_F_FRAGLIST | NETIF_F_HW_CSUM | \ diff --git a/drivers/net/wireless/ath/ath10k/bmi.c b/drivers/net/wireless/ath/ath10k/bmi.c index af6546572df26b..9a4f8e815412cb 100644 --- a/drivers/net/wireless/ath/ath10k/bmi.c +++ b/drivers/net/wireless/ath/ath10k/bmi.c @@ -2,6 +2,7 @@ /* * Copyright (c) 2005-2011 Atheros Communications Inc. * Copyright (c) 2011-2014,2016-2017 Qualcomm Atheros, Inc. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #include "bmi.h" diff --git a/drivers/net/wireless/ath/ath10k/ce.c b/drivers/net/wireless/ath/ath10k/ce.c index c27b8204718a6d..afae4a8027f833 100644 --- a/drivers/net/wireless/ath/ath10k/ce.c +++ b/drivers/net/wireless/ath/ath10k/ce.c @@ -3,6 +3,7 @@ * Copyright (c) 2005-2011 Atheros Communications Inc. * Copyright (c) 2011-2017 Qualcomm Atheros, Inc. * Copyright (c) 2018 The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #include "hif.h" diff --git a/drivers/net/wireless/ath/ath10k/core.c b/drivers/net/wireless/ath/ath10k/core.c index 6cdb225b7eaccc..0032f8aa892ffa 100644 --- a/drivers/net/wireless/ath/ath10k/core.c +++ b/drivers/net/wireless/ath/ath10k/core.c @@ -3,6 +3,7 @@ * Copyright (c) 2005-2011 Atheros Communications Inc. * Copyright (c) 2011-2017 Qualcomm Atheros, Inc. * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include @@ -100,6 +101,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = { .hw_restart_disconnect = false, .use_fw_tx_credits = true, .delay_unmap_buffer = false, + .mcast_frame_registration = false, }, { .id = QCA988X_HW_2_0_VERSION, @@ -140,6 +142,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = { .hw_restart_disconnect = false, .use_fw_tx_credits = true, .delay_unmap_buffer = false, + .mcast_frame_registration = false, }, { .id = QCA9887_HW_1_0_VERSION, @@ -181,6 +184,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = { .hw_restart_disconnect = false, .use_fw_tx_credits = true, .delay_unmap_buffer = false, + .mcast_frame_registration = false, }, { .id = QCA6174_HW_3_2_VERSION, @@ -217,6 +221,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = { .hw_restart_disconnect = false, .use_fw_tx_credits = true, .delay_unmap_buffer = false, + .mcast_frame_registration = false, }, { .id = QCA6174_HW_2_1_VERSION, @@ -257,6 +262,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = { .hw_restart_disconnect = false, .use_fw_tx_credits = true, .delay_unmap_buffer = false, + .mcast_frame_registration = false, }, { .id = QCA6174_HW_2_1_VERSION, @@ -297,6 +303,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = { .hw_restart_disconnect = false, .use_fw_tx_credits = true, .delay_unmap_buffer = false, + .mcast_frame_registration = false, }, { .id = QCA6174_HW_3_0_VERSION, @@ -337,6 +344,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = { .hw_restart_disconnect = false, .use_fw_tx_credits = true, .delay_unmap_buffer = false, + .mcast_frame_registration = false, }, { .id = QCA6174_HW_3_2_VERSION, @@ -381,6 +389,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = { .hw_restart_disconnect = false, .use_fw_tx_credits = true, .delay_unmap_buffer = false, + .mcast_frame_registration = true, }, { .id = QCA99X0_HW_2_0_DEV_VERSION, @@ -427,6 +436,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = { .hw_restart_disconnect = false, .use_fw_tx_credits = true, .delay_unmap_buffer = false, + .mcast_frame_registration = false, }, { .id = QCA9984_HW_1_0_DEV_VERSION, @@ -480,6 +490,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = { .hw_restart_disconnect = false, .use_fw_tx_credits = true, .delay_unmap_buffer = false, + .mcast_frame_registration = false, }, { .id = QCA9888_HW_2_0_DEV_VERSION, @@ -530,6 +541,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = { .hw_restart_disconnect = false, .use_fw_tx_credits = true, .delay_unmap_buffer = false, + .mcast_frame_registration = false, }, { .id = QCA9377_HW_1_0_DEV_VERSION, @@ -570,6 +582,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = { .hw_restart_disconnect = false, .use_fw_tx_credits = true, .delay_unmap_buffer = false, + .mcast_frame_registration = false, }, { .id = QCA9377_HW_1_1_DEV_VERSION, @@ -612,6 +625,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = { .hw_restart_disconnect = false, .use_fw_tx_credits = true, .delay_unmap_buffer = false, + .mcast_frame_registration = false, }, { .id = QCA9377_HW_1_1_DEV_VERSION, @@ -645,6 +659,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = { .hw_restart_disconnect = false, .use_fw_tx_credits = true, .delay_unmap_buffer = false, + .mcast_frame_registration = false, }, { .id = QCA4019_HW_1_0_DEV_VERSION, @@ -692,6 +707,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = { .hw_restart_disconnect = false, .use_fw_tx_credits = true, .delay_unmap_buffer = false, + .mcast_frame_registration = false, }, { .id = WCN3990_HW_1_0_DEV_VERSION, @@ -725,6 +741,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = { .hw_restart_disconnect = true, .use_fw_tx_credits = false, .delay_unmap_buffer = true, + .mcast_frame_registration = false, }, }; diff --git a/drivers/net/wireless/ath/ath10k/core.h b/drivers/net/wireless/ath/ath10k/core.h index 4b5239de401840..c110d15528bd05 100644 --- a/drivers/net/wireless/ath/ath10k/core.h +++ b/drivers/net/wireless/ath/ath10k/core.h @@ -3,6 +3,7 @@ * Copyright (c) 2005-2011 Atheros Communications Inc. * Copyright (c) 2011-2017 Qualcomm Atheros, Inc. * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef _CORE_H_ @@ -607,7 +608,7 @@ struct ath10k_vif { u8 tim_bitmap[64]; u8 tim_len; u32 ssid_len; - u8 ssid[IEEE80211_MAX_SSID_LEN]; + u8 ssid[IEEE80211_MAX_SSID_LEN] __nonstring; bool hidden_ssid; /* P2P_IE with NoA attribute for P2P_GO case */ u32 noa_len; diff --git a/drivers/net/wireless/ath/ath10k/coredump.c b/drivers/net/wireless/ath/ath10k/coredump.c index 2d1634a890dde3..bb3a276b7ed584 100644 --- a/drivers/net/wireless/ath/ath10k/coredump.c +++ b/drivers/net/wireless/ath/ath10k/coredump.c @@ -2,6 +2,7 @@ /* * Copyright (c) 2011-2017 Qualcomm Atheros, Inc. * Copyright (c) 2018, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #include "coredump.h" diff --git a/drivers/net/wireless/ath/ath10k/coredump.h b/drivers/net/wireless/ath/ath10k/coredump.h index 437b9759f05d3d..e5ef0352e319c7 100644 --- a/drivers/net/wireless/ath/ath10k/coredump.h +++ b/drivers/net/wireless/ath/ath10k/coredump.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: ISC */ /* * Copyright (c) 2011-2017 Qualcomm Atheros, Inc. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef _COREDUMP_H_ diff --git a/drivers/net/wireless/ath/ath10k/debug.c b/drivers/net/wireless/ath/ath10k/debug.c index ad9cf953a2fcf2..b93a64bf819018 100644 --- a/drivers/net/wireless/ath/ath10k/debug.c +++ b/drivers/net/wireless/ath/ath10k/debug.c @@ -3,6 +3,7 @@ * Copyright (c) 2005-2011 Atheros Communications Inc. * Copyright (c) 2011-2017 Qualcomm Atheros, Inc. * Copyright (c) 2018, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #include diff --git a/drivers/net/wireless/ath/ath10k/debugfs_sta.c b/drivers/net/wireless/ath/ath10k/debugfs_sta.c index 87a3365330ff80..394bf3c32abff6 100644 --- a/drivers/net/wireless/ath/ath10k/debugfs_sta.c +++ b/drivers/net/wireless/ath/ath10k/debugfs_sta.c @@ -2,6 +2,7 @@ /* * Copyright (c) 2014-2017 Qualcomm Atheros, Inc. * Copyright (c) 2018, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #include "core.h" diff --git a/drivers/net/wireless/ath/ath10k/htc.c b/drivers/net/wireless/ath/ath10k/htc.c index 5bfeecb95fca23..a6e21ce90bad64 100644 --- a/drivers/net/wireless/ath/ath10k/htc.c +++ b/drivers/net/wireless/ath/ath10k/htc.c @@ -2,6 +2,7 @@ /* * Copyright (c) 2005-2011 Atheros Communications Inc. * Copyright (c) 2011-2017 Qualcomm Atheros, Inc. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #include "core.h" diff --git a/drivers/net/wireless/ath/ath10k/htc.h b/drivers/net/wireless/ath/ath10k/htc.h index 0d180faf3b77f7..7ff665020015d4 100644 --- a/drivers/net/wireless/ath/ath10k/htc.h +++ b/drivers/net/wireless/ath/ath10k/htc.h @@ -246,26 +246,12 @@ struct ath10k_htc_lookahead_bundle { struct ath10k_htc_record { struct ath10k_ath10k_htc_record_hdr hdr; union { - struct ath10k_htc_credit_report credit_report[0]; - struct ath10k_htc_lookahead_report lookahead_report[0]; - struct ath10k_htc_lookahead_bundle lookahead_bundle[0]; - u8 pauload[0]; + DECLARE_FLEX_ARRAY(struct ath10k_htc_credit_report, credit_report); + DECLARE_FLEX_ARRAY(struct ath10k_htc_lookahead_report, lookahead_report); + DECLARE_FLEX_ARRAY(struct ath10k_htc_lookahead_bundle, lookahead_bundle); }; } __packed __aligned(4); -/* - * note: the trailer offset is dynamic depending - * on payload length. this is only a struct layout draft - */ -struct ath10k_htc_frame { - struct ath10k_htc_hdr hdr; - union { - struct ath10k_htc_msg msg; - u8 payload[0]; - }; - struct ath10k_htc_record trailer[0]; -} __packed __aligned(4); - /*******************/ /* Host-side stuff */ /*******************/ diff --git a/drivers/net/wireless/ath/ath10k/htt.h b/drivers/net/wireless/ath/ath10k/htt.h index c80470e8886a03..4a9270e2a4c882 100644 --- a/drivers/net/wireless/ath/ath10k/htt.h +++ b/drivers/net/wireless/ath/ath10k/htt.h @@ -3,6 +3,7 @@ * Copyright (c) 2005-2011 Atheros Communications Inc. * Copyright (c) 2011-2017 Qualcomm Atheros, Inc. * Copyright (c) 2018, The Linux Foundation. All rights reserved. + * Copyright (c) 2021, 2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef _HTT_H_ diff --git a/drivers/net/wireless/ath/ath10k/htt_rx.c b/drivers/net/wireless/ath/ath10k/htt_rx.c index b261d6371c0f3a..fa0f598ed6bfef 100644 --- a/drivers/net/wireless/ath/ath10k/htt_rx.c +++ b/drivers/net/wireless/ath/ath10k/htt_rx.c @@ -3,6 +3,7 @@ * Copyright (c) 2005-2011 Atheros Communications Inc. * Copyright (c) 2011-2017 Qualcomm Atheros, Inc. * Copyright (c) 2018, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #include "core.h" diff --git a/drivers/net/wireless/ath/ath10k/htt_tx.c b/drivers/net/wireless/ath/ath10k/htt_tx.c index be4d4536aaa876..9725feecefd6fb 100644 --- a/drivers/net/wireless/ath/ath10k/htt_tx.c +++ b/drivers/net/wireless/ath/ath10k/htt_tx.c @@ -2,6 +2,7 @@ /* * Copyright (c) 2005-2011 Atheros Communications Inc. * Copyright (c) 2011-2017 Qualcomm Atheros, Inc. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #include @@ -40,7 +41,6 @@ static void __ath10k_htt_tx_txq_recalc(struct ieee80211_hw *hw, struct ath10k *ar = hw->priv; struct ath10k_sta *arsta; struct ath10k_vif *arvif = (void *)txq->vif->drv_priv; - unsigned long frame_cnt; unsigned long byte_cnt; int idx; u32 bit; @@ -67,7 +67,7 @@ static void __ath10k_htt_tx_txq_recalc(struct ieee80211_hw *hw, bit = BIT(peer_id % 32); idx = peer_id / 32; - ieee80211_txq_get_depth(txq, &frame_cnt, &byte_cnt); + ieee80211_txq_get_depth(txq, NULL, &byte_cnt); count = ath10k_htt_tx_txq_calc_size(byte_cnt); if (unlikely(peer_id >= ar->htt.tx_q_state.num_peers) || diff --git a/drivers/net/wireless/ath/ath10k/hw.c b/drivers/net/wireless/ath/ath10k/hw.c index 6d32b43a4da65e..8fafe096adff53 100644 --- a/drivers/net/wireless/ath/ath10k/hw.c +++ b/drivers/net/wireless/ath/ath10k/hw.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: ISC /* * Copyright (c) 2014-2017 Qualcomm Atheros, Inc. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #include diff --git a/drivers/net/wireless/ath/ath10k/hw.h b/drivers/net/wireless/ath/ath10k/hw.h index 9643031a4427ad..93c0730919966d 100644 --- a/drivers/net/wireless/ath/ath10k/hw.h +++ b/drivers/net/wireless/ath/ath10k/hw.h @@ -3,6 +3,7 @@ * Copyright (c) 2005-2011 Atheros Communications Inc. * Copyright (c) 2011-2017 Qualcomm Atheros, Inc. * Copyright (c) 2018 The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef _HW_H_ @@ -639,6 +640,9 @@ struct ath10k_hw_params { bool use_fw_tx_credits; bool delay_unmap_buffer; + + /* The hardware support multicast frame registrations */ + bool mcast_frame_registration; }; struct htt_resp; diff --git a/drivers/net/wireless/ath/ath10k/mac.c b/drivers/net/wireless/ath/ath10k/mac.c index 2cf693f3fea96e..090bcf148d0c75 100644 --- a/drivers/net/wireless/ath/ath10k/mac.c +++ b/drivers/net/wireless/ath/ath10k/mac.c @@ -3,6 +3,7 @@ * Copyright (c) 2005-2011 Atheros Communications Inc. * Copyright (c) 2011-2017 Qualcomm Atheros, Inc. * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include "mac.h" @@ -1242,7 +1243,7 @@ static bool ath10k_mac_monitor_vdev_is_needed(struct ath10k *ar) return ar->monitor || (!test_bit(ATH10K_FW_FEATURE_ALLOWS_MESH_BCAST, ar->running_fw->fw_file.fw_features) && - (ar->filter_flags & FIF_OTHER_BSS)) || + (ar->filter_flags & (FIF_OTHER_BSS | FIF_MCAST_ACTION))) || test_bit(ATH10K_CAC_RUNNING, &ar->dev_flags); } @@ -6025,10 +6026,15 @@ static void ath10k_configure_filter(struct ieee80211_hw *hw, { struct ath10k *ar = hw->priv; int ret; + unsigned int supported = SUPPORTED_FILTERS; mutex_lock(&ar->conf_mutex); - *total_flags &= SUPPORTED_FILTERS; + if (ar->hw_params.mcast_frame_registration) + supported |= FIF_MCAST_ACTION; + + *total_flags &= supported; + ar->filter_flags = *total_flags; ret = ath10k_monitor_recalc(ar); @@ -6121,9 +6127,8 @@ static void ath10k_bss_info_changed(struct ieee80211_hw *hw, if (ieee80211_vif_is_mesh(vif)) { /* mesh doesn't use SSID but firmware needs it */ - strncpy(arvif->u.ap.ssid, "mesh", - sizeof(arvif->u.ap.ssid)); arvif->u.ap.ssid_len = 4; + memcpy(arvif->u.ap.ssid, "mesh", arvif->u.ap.ssid_len); } } @@ -10118,6 +10123,10 @@ int ath10k_mac_register(struct ath10k *ar) NL80211_EXT_FEATURE_SET_SCAN_DWELL); wiphy_ext_feature_set(ar->hw->wiphy, NL80211_EXT_FEATURE_AQL); + if (ar->hw_params.mcast_frame_registration) + wiphy_ext_feature_set(ar->hw->wiphy, + NL80211_EXT_FEATURE_MULTICAST_REGISTRATIONS); + if (test_bit(WMI_SERVICE_TX_DATA_ACK_RSSI, ar->wmi.svc_map) || test_bit(WMI_SERVICE_HTT_MGMT_TX_COMP_VALID_FLAGS, ar->wmi.svc_map)) wiphy_ext_feature_set(ar->hw->wiphy, diff --git a/drivers/net/wireless/ath/ath10k/pci.c b/drivers/net/wireless/ath/ath10k/pci.c index 2f8c785277af5e..3de2de6d44bc9c 100644 --- a/drivers/net/wireless/ath/ath10k/pci.c +++ b/drivers/net/wireless/ath/ath10k/pci.c @@ -2,6 +2,7 @@ /* * Copyright (c) 2005-2011 Atheros Communications Inc. * Copyright (c) 2011-2017 Qualcomm Atheros, Inc. + * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include diff --git a/drivers/net/wireless/ath/ath10k/pci.h b/drivers/net/wireless/ath/ath10k/pci.h index 480cd97ab739de..27bb4cf2dfea93 100644 --- a/drivers/net/wireless/ath/ath10k/pci.h +++ b/drivers/net/wireless/ath/ath10k/pci.h @@ -2,6 +2,7 @@ /* * Copyright (c) 2005-2011 Atheros Communications Inc. * Copyright (c) 2011-2017 Qualcomm Atheros, Inc. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef _PCI_H_ diff --git a/drivers/net/wireless/ath/ath10k/qmi.c b/drivers/net/wireless/ath/ath10k/qmi.c index 52c1a3de8da60a..38e939f572a9ed 100644 --- a/drivers/net/wireless/ath/ath10k/qmi.c +++ b/drivers/net/wireless/ath/ath10k/qmi.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: ISC /* * Copyright (c) 2018 The Linux Foundation. All rights reserved. + * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include diff --git a/drivers/net/wireless/ath/ath10k/qmi_wlfw_v01.c b/drivers/net/wireless/ath/ath10k/qmi_wlfw_v01.c index 1c81e454f943fd..0e85c75d227836 100644 --- a/drivers/net/wireless/ath/ath10k/qmi_wlfw_v01.c +++ b/drivers/net/wireless/ath/ath10k/qmi_wlfw_v01.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: ISC /* * Copyright (c) 2018 The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #include diff --git a/drivers/net/wireless/ath/ath10k/qmi_wlfw_v01.h b/drivers/net/wireless/ath/ath10k/qmi_wlfw_v01.h index f0db991408dc26..9f311f3bc9e7f9 100644 --- a/drivers/net/wireless/ath/ath10k/qmi_wlfw_v01.h +++ b/drivers/net/wireless/ath/ath10k/qmi_wlfw_v01.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: ISC */ /* * Copyright (c) 2018 The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef WCN3990_QMI_SVC_V01_H diff --git a/drivers/net/wireless/ath/ath10k/rx_desc.h b/drivers/net/wireless/ath/ath10k/rx_desc.h index 777e53aa69dc86..564293df1e9acf 100644 --- a/drivers/net/wireless/ath/ath10k/rx_desc.h +++ b/drivers/net/wireless/ath/ath10k/rx_desc.h @@ -2,6 +2,7 @@ /* * Copyright (c) 2005-2011 Atheros Communications Inc. * Copyright (c) 2011-2017 Qualcomm Atheros, Inc. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef _RX_DESC_H_ diff --git a/drivers/net/wireless/ath/ath10k/sdio.c b/drivers/net/wireless/ath/ath10k/sdio.c index 56fbcfb80bf886..0ab5433f6cf6f2 100644 --- a/drivers/net/wireless/ath/ath10k/sdio.c +++ b/drivers/net/wireless/ath/ath10k/sdio.c @@ -3,6 +3,7 @@ * Copyright (c) 2004-2011 Atheros Communications Inc. * Copyright (c) 2011-2012,2017 Qualcomm Atheros, Inc. * Copyright (c) 2016-2017 Erik Stromdahl + * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include diff --git a/drivers/net/wireless/ath/ath10k/thermal.c b/drivers/net/wireless/ath/ath10k/thermal.c index cefd97323dfe50..31c8d7fbb0955b 100644 --- a/drivers/net/wireless/ath/ath10k/thermal.c +++ b/drivers/net/wireless/ath/ath10k/thermal.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: ISC /* * Copyright (c) 2014-2015 Qualcomm Atheros, Inc. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #include diff --git a/drivers/net/wireless/ath/ath10k/usb.h b/drivers/net/wireless/ath/ath10k/usb.h index 48e066ba816249..7e4cfbb673c9a8 100644 --- a/drivers/net/wireless/ath/ath10k/usb.h +++ b/drivers/net/wireless/ath/ath10k/usb.h @@ -3,6 +3,7 @@ * Copyright (c) 2004-2011 Atheros Communications Inc. * Copyright (c) 2011-2012 Qualcomm Atheros, Inc. * Copyright (c) 2016-2017 Erik Stromdahl + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef _USB_H_ diff --git a/drivers/net/wireless/ath/ath10k/wmi-tlv.h b/drivers/net/wireless/ath/ath10k/wmi-tlv.h index dbb48d70f2e93e..83a8f07a687f73 100644 --- a/drivers/net/wireless/ath/ath10k/wmi-tlv.h +++ b/drivers/net/wireless/ath/ath10k/wmi-tlv.h @@ -3,6 +3,7 @@ * Copyright (c) 2005-2011 Atheros Communications Inc. * Copyright (c) 2011-2017 Qualcomm Atheros, Inc. * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef _WMI_TLV_H #define _WMI_TLV_H diff --git a/drivers/net/wireless/ath/ath10k/wmi.c b/drivers/net/wireless/ath/ath10k/wmi.c index 05fa7d4c0e1aba..88befe92f95dcc 100644 --- a/drivers/net/wireless/ath/ath10k/wmi.c +++ b/drivers/net/wireless/ath/ath10k/wmi.c @@ -3,6 +3,7 @@ * Copyright (c) 2005-2011 Atheros Communications Inc. * Copyright (c) 2011-2017 Qualcomm Atheros, Inc. * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #include diff --git a/drivers/net/wireless/ath/ath10k/wmi.h b/drivers/net/wireless/ath/ath10k/wmi.h index b112e88260931c..9146df98fceeee 100644 --- a/drivers/net/wireless/ath/ath10k/wmi.h +++ b/drivers/net/wireless/ath/ath10k/wmi.h @@ -3,6 +3,7 @@ * Copyright (c) 2005-2011 Atheros Communications Inc. * Copyright (c) 2011-2017 Qualcomm Atheros, Inc. * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef _WMI_H_ diff --git a/drivers/net/wireless/ath/ath10k/wow.c b/drivers/net/wireless/ath/ath10k/wow.c index 20b9aa8ddf7d52..aa7b2e703f3d4b 100644 --- a/drivers/net/wireless/ath/ath10k/wow.c +++ b/drivers/net/wireless/ath/ath10k/wow.c @@ -2,6 +2,7 @@ /* * Copyright (c) 2015-2017 Qualcomm Atheros, Inc. * Copyright (c) 2018, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #include "mac.h" diff --git a/drivers/net/wireless/ath/ath11k/Kconfig b/drivers/net/wireless/ath/ath11k/Kconfig index ad5cc6cac05bc9..27f0523bf96754 100644 --- a/drivers/net/wireless/ath/ath11k/Kconfig +++ b/drivers/net/wireless/ath/ath11k/Kconfig @@ -2,7 +2,7 @@ config ATH11K tristate "Qualcomm Technologies 802.11ax chipset support" depends on MAC80211 && HAS_DMA - depends on CRYPTO_MICHAEL_MIC + select CRYPTO_MICHAEL_MIC select ATH_COMMON select QCOM_QMI_HELPERS help diff --git a/drivers/net/wireless/ath/ath11k/ahb.c b/drivers/net/wireless/ath/ath11k/ahb.c index 235336ef2a7a59..7c0a235179497d 100644 --- a/drivers/net/wireless/ath/ath11k/ahb.c +++ b/drivers/net/wireless/ath/ath11k/ahb.c @@ -803,8 +803,8 @@ static int ath11k_core_get_rproc(struct ath11k_base *ab) prproc = rproc_get_by_phandle(rproc_phandle); if (!prproc) { - ath11k_err(ab, "failed to get rproc\n"); - return -EINVAL; + ath11k_dbg(ab, ATH11K_DBG_AHB, "failed to get rproc, deferring\n"); + return -EPROBE_DEFER; } ab_ahb->tgt_rproc = prproc; @@ -1251,7 +1251,7 @@ static void ath11k_ahb_free_resources(struct ath11k_base *ab) platform_set_drvdata(pdev, NULL); } -static int ath11k_ahb_remove(struct platform_device *pdev) +static void ath11k_ahb_remove(struct platform_device *pdev) { struct ath11k_base *ab = platform_get_drvdata(pdev); @@ -1267,8 +1267,6 @@ static int ath11k_ahb_remove(struct platform_device *pdev) qmi_fail: ath11k_ahb_free_resources(ab); - - return 0; } static void ath11k_ahb_shutdown(struct platform_device *pdev) @@ -1296,7 +1294,7 @@ static struct platform_driver ath11k_ahb_driver = { .of_match_table = ath11k_ahb_of_match, }, .probe = ath11k_ahb_probe, - .remove = ath11k_ahb_remove, + .remove_new = ath11k_ahb_remove, .shutdown = ath11k_ahb_shutdown, }; diff --git a/drivers/net/wireless/ath/ath11k/ce.c b/drivers/net/wireless/ath/ath11k/ce.c index 289d47ae92afc5..e66e86bdec20ff 100644 --- a/drivers/net/wireless/ath/ath11k/ce.c +++ b/drivers/net/wireless/ath/ath11k/ce.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. - * Copyright (c) 2021, Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include "dp_rx.h" diff --git a/drivers/net/wireless/ath/ath11k/ce.h b/drivers/net/wireless/ath/ath11k/ce.h index c0f6a0ba86df09..69946fc700777b 100644 --- a/drivers/net/wireless/ath/ath11k/ce.h +++ b/drivers/net/wireless/ath/ath11k/ce.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef ATH11K_CE_H diff --git a/drivers/net/wireless/ath/ath11k/core.h b/drivers/net/wireless/ath/ath11k/core.h index f12b606e2d2e56..7e3b6779f4e969 100644 --- a/drivers/net/wireless/ath/ath11k/core.h +++ b/drivers/net/wireless/ath/ath11k/core.h @@ -599,7 +599,6 @@ struct ath11k { struct ath11k_base *ab; struct ath11k_pdev *pdev; struct ieee80211_hw *hw; - struct ieee80211_ops *ops; struct ath11k_pdev_wmi *wmi; struct ath11k_pdev_dp dp; u8 mac_addr[ETH_ALEN]; diff --git a/drivers/net/wireless/ath/ath11k/dbring.c b/drivers/net/wireless/ath/ath11k/dbring.c index 5536e864233124..fbb6e8d8a47692 100644 --- a/drivers/net/wireless/ath/ath11k/dbring.c +++ b/drivers/net/wireless/ath/ath11k/dbring.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2019-2020 The Linux Foundation. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include "core.h" diff --git a/drivers/net/wireless/ath/ath11k/dbring.h b/drivers/net/wireless/ath/ath11k/dbring.h index ef906c687b8cdb..2f93b78a50df0e 100644 --- a/drivers/net/wireless/ath/ath11k/dbring.h +++ b/drivers/net/wireless/ath/ath11k/dbring.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2019-2020 The Linux Foundation. All rights reserved. + * Copyright (c) 2021 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef ATH11K_DBRING_H diff --git a/drivers/net/wireless/ath/ath11k/debug.c b/drivers/net/wireless/ath/ath11k/debug.c index f5c8a34c8802f0..2b8544355fc1a6 100644 --- a/drivers/net/wireless/ath/ath11k/debug.c +++ b/drivers/net/wireless/ath/ath11k/debug.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. + * Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include diff --git a/drivers/net/wireless/ath/ath11k/debug.h b/drivers/net/wireless/ath/ath11k/debug.h index 9c52804ef8ac30..cc8934d156977c 100644 --- a/drivers/net/wireless/ath/ath11k/debug.h +++ b/drivers/net/wireless/ath/ath11k/debug.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. - * Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef _ATH11K_DEBUG_H_ diff --git a/drivers/net/wireless/ath/ath11k/debugfs.c b/drivers/net/wireless/ath/ath11k/debugfs.c index be76e7d1c4366e..a847bc0d50c0f0 100644 --- a/drivers/net/wireless/ath/ath11k/debugfs.c +++ b/drivers/net/wireless/ath/ath11k/debugfs.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2020 The Linux Foundation. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include diff --git a/drivers/net/wireless/ath/ath11k/debugfs.h b/drivers/net/wireless/ath/ath11k/debugfs.h index 3af0169f6cf218..44d15845f39a67 100644 --- a/drivers/net/wireless/ath/ath11k/debugfs.h +++ b/drivers/net/wireless/ath/ath11k/debugfs.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. + * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef _ATH11K_DEBUGFS_H_ diff --git a/drivers/net/wireless/ath/ath11k/debugfs_htt_stats.c b/drivers/net/wireless/ath/ath11k/debugfs_htt_stats.c index 0207fc4910f342..870e86a31bf896 100644 --- a/drivers/net/wireless/ath/ath11k/debugfs_htt_stats.c +++ b/drivers/net/wireless/ath/ath11k/debugfs_htt_stats.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. - * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include diff --git a/drivers/net/wireless/ath/ath11k/debugfs_htt_stats.h b/drivers/net/wireless/ath/ath11k/debugfs_htt_stats.h index 96219301f05bd4..476689bbd4dad7 100644 --- a/drivers/net/wireless/ath/ath11k/debugfs_htt_stats.h +++ b/drivers/net/wireless/ath/ath11k/debugfs_htt_stats.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. - * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef DEBUG_HTT_STATS_H diff --git a/drivers/net/wireless/ath/ath11k/debugfs_sta.c b/drivers/net/wireless/ath/ath11k/debugfs_sta.c index 8c177fba6f145a..f56a24b6c8da21 100644 --- a/drivers/net/wireless/ath/ath11k/debugfs_sta.c +++ b/drivers/net/wireless/ath/ath11k/debugfs_sta.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include diff --git a/drivers/net/wireless/ath/ath11k/debugfs_sta.h b/drivers/net/wireless/ath/ath11k/debugfs_sta.h index e6c11b3a40aa93..ace877e19275eb 100644 --- a/drivers/net/wireless/ath/ath11k/debugfs_sta.h +++ b/drivers/net/wireless/ath/ath11k/debugfs_sta.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2020 The Linux Foundation. All rights reserved. + * Copyright (c) 2021 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef _ATH11K_DEBUGFS_STA_H_ diff --git a/drivers/net/wireless/ath/ath11k/dp.c b/drivers/net/wireless/ath/ath11k/dp.c index a7252b52555cce..8975dc57ad7768 100644 --- a/drivers/net/wireless/ath/ath11k/dp.c +++ b/drivers/net/wireless/ath/ath11k/dp.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. - * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include diff --git a/drivers/net/wireless/ath/ath11k/dp.h b/drivers/net/wireless/ath/ath11k/dp.h index 15815af453b2a6..2f6dd69d3be276 100644 --- a/drivers/net/wireless/ath/ath11k/dp.h +++ b/drivers/net/wireless/ath/ath11k/dp.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. - * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef ATH11K_DP_H diff --git a/drivers/net/wireless/ath/ath11k/dp_rx.c b/drivers/net/wireless/ath/ath11k/dp_rx.c index 7eac93ce7a1dd0..afd481f5858f03 100644 --- a/drivers/net/wireless/ath/ath11k/dp_rx.c +++ b/drivers/net/wireless/ath/ath11k/dp_rx.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include diff --git a/drivers/net/wireless/ath/ath11k/dp_tx.c b/drivers/net/wireless/ath/ath11k/dp_tx.c index a5fa08bc623b85..c1072e66e3e8fd 100644 --- a/drivers/net/wireless/ath/ath11k/dp_tx.c +++ b/drivers/net/wireless/ath/ath11k/dp_tx.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. - * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include "core.h" diff --git a/drivers/net/wireless/ath/ath11k/dp_tx.h b/drivers/net/wireless/ath/ath11k/dp_tx.h index 68a21ea9b93463..61be2265e09f08 100644 --- a/drivers/net/wireless/ath/ath11k/dp_tx.h +++ b/drivers/net/wireless/ath/ath11k/dp_tx.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. + * Copyright (c) 2021, 2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef ATH11K_DP_TX_H diff --git a/drivers/net/wireless/ath/ath11k/fw.c b/drivers/net/wireless/ath/ath11k/fw.c index 8f84fba29886eb..4e36292a79db89 100644 --- a/drivers/net/wireless/ath/ath11k/fw.c +++ b/drivers/net/wireless/ath/ath11k/fw.c @@ -133,7 +133,7 @@ static int ath11k_fw_request_firmware_api_n(struct ath11k_base *ab, len -= ie_len; data += ie_len; - }; + } return 0; diff --git a/drivers/net/wireless/ath/ath11k/hal.c b/drivers/net/wireless/ath/ath11k/hal.c index 23f3af8e372df8..c060c4b5c0cc36 100644 --- a/drivers/net/wireless/ath/ath11k/hal.c +++ b/drivers/net/wireless/ath/ath11k/hal.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. - * Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include #include "hal_tx.h" diff --git a/drivers/net/wireless/ath/ath11k/hal.h b/drivers/net/wireless/ath/ath11k/hal.h index 1942d41d6de541..80447f488954a8 100644 --- a/drivers/net/wireless/ath/ath11k/hal.h +++ b/drivers/net/wireless/ath/ath11k/hal.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. - * Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef ATH11K_HAL_H diff --git a/drivers/net/wireless/ath/ath11k/hal_desc.h b/drivers/net/wireless/ath/ath11k/hal_desc.h index d895ea878d9f03..b2fd180bd28e6b 100644 --- a/drivers/net/wireless/ath/ath11k/hal_desc.h +++ b/drivers/net/wireless/ath/ath11k/hal_desc.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. + * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #include "core.h" diff --git a/drivers/net/wireless/ath/ath11k/hal_rx.c b/drivers/net/wireless/ath/ath11k/hal_rx.c index 41946795d6203b..e758ee8e17c904 100644 --- a/drivers/net/wireless/ath/ath11k/hal_rx.c +++ b/drivers/net/wireless/ath/ath11k/hal_rx.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include "debug.h" diff --git a/drivers/net/wireless/ath/ath11k/hal_rx.h b/drivers/net/wireless/ath/ath11k/hal_rx.h index 472a52cf5889f2..0fa9aef9d533b8 100644 --- a/drivers/net/wireless/ath/ath11k/hal_rx.h +++ b/drivers/net/wireless/ath/ath11k/hal_rx.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef ATH11K_HAL_RX_H diff --git a/drivers/net/wireless/ath/ath11k/hif.h b/drivers/net/wireless/ath/ath11k/hif.h index d68ed4214dec10..877a4073fed69c 100644 --- a/drivers/net/wireless/ath/ath11k/hif.h +++ b/drivers/net/wireless/ath/ath11k/hif.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2019-2020 The Linux Foundation. All rights reserved. + * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef _HIF_H_ diff --git a/drivers/net/wireless/ath/ath11k/htc.c b/drivers/net/wireless/ath/ath11k/htc.c index 2c2e425c866595..23054ab29a5eed 100644 --- a/drivers/net/wireless/ath/ath11k/htc.c +++ b/drivers/net/wireless/ath/ath11k/htc.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. + * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include #include diff --git a/drivers/net/wireless/ath/ath11k/htc.h b/drivers/net/wireless/ath/ath11k/htc.h index d31e501c807cef..86f77eacaea7a3 100644 --- a/drivers/net/wireless/ath/ath11k/htc.h +++ b/drivers/net/wireless/ath/ath11k/htc.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. + * Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef ATH11K_HTC_H @@ -150,10 +151,7 @@ struct ath11k_htc_credit_report { struct ath11k_htc_record { struct ath11k_htc_record_hdr hdr; - union { - struct ath11k_htc_credit_report credit_report[0]; - u8 pauload[0]; - }; + struct ath11k_htc_credit_report credit_report[]; } __packed __aligned(4); enum ath11k_htc_svc_gid { diff --git a/drivers/net/wireless/ath/ath11k/hw.c b/drivers/net/wireless/ath/ath11k/hw.c index d7b5ec6e690490..77d8f9237680b2 100644 --- a/drivers/net/wireless/ath/ath11k/hw.c +++ b/drivers/net/wireless/ath/ath11k/hw.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2020 The Linux Foundation. All rights reserved. - * Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include diff --git a/drivers/net/wireless/ath/ath11k/hw.h b/drivers/net/wireless/ath/ath11k/hw.h index d51a99669dd6ee..1b070747a5dbfe 100644 --- a/drivers/net/wireless/ath/ath11k/hw.h +++ b/drivers/net/wireless/ath/ath11k/hw.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022, Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef ATH11K_HW_H diff --git a/drivers/net/wireless/ath/ath11k/mac.c b/drivers/net/wireless/ath/ath11k/mac.c index 7f7b3981777377..db241589424d51 100644 --- a/drivers/net/wireless/ath/ath11k/mac.c +++ b/drivers/net/wireless/ath/ath11k/mac.c @@ -4654,6 +4654,14 @@ static int ath11k_station_disassoc(struct ath11k *ar, return 0; } +static u32 ath11k_mac_max_nss(const u8 *ht_mcs_mask, const u16 *vht_mcs_mask, + const u16 *he_mcs_mask) +{ + return max3(ath11k_mac_max_ht_nss(ht_mcs_mask), + ath11k_mac_max_vht_nss(vht_mcs_mask), + ath11k_mac_max_he_nss(he_mcs_mask)); +} + static void ath11k_sta_rc_update_wk(struct work_struct *wk) { struct ath11k *ar; @@ -4699,9 +4707,7 @@ static void ath11k_sta_rc_update_wk(struct work_struct *wk) mutex_lock(&ar->conf_mutex); nss = max_t(u32, 1, nss); - nss = min(nss, max(max(ath11k_mac_max_ht_nss(ht_mcs_mask), - ath11k_mac_max_vht_nss(vht_mcs_mask)), - ath11k_mac_max_he_nss(he_mcs_mask))); + nss = min(nss, ath11k_mac_max_nss(ht_mcs_mask, vht_mcs_mask, he_mcs_mask)); if (changed & IEEE80211_RC_BW_CHANGED) { /* Get the peer phymode */ @@ -8391,9 +8397,7 @@ ath11k_mac_op_set_bitrate_mask(struct ieee80211_hw *hw, ath11k_warn(ar->ab, "could not update fixed rate settings to all peers due to mcs/nss incompatibility\n"); nss = min_t(u32, ar->num_tx_chains, - max(max(ath11k_mac_max_ht_nss(ht_mcs_mask), - ath11k_mac_max_vht_nss(vht_mcs_mask)), - ath11k_mac_max_he_nss(he_mcs_mask))); + ath11k_mac_max_nss(ht_mcs_mask, vht_mcs_mask, he_mcs_mask)); /* If multiple rates across different preambles are given * we can reconfigure this info with all peers using PEER_ASSOC diff --git a/drivers/net/wireless/ath/ath11k/mac.h b/drivers/net/wireless/ath/ath11k/mac.h index 0231783ad754be..0dfdeed5177b88 100644 --- a/drivers/net/wireless/ath/ath11k/mac.h +++ b/drivers/net/wireless/ath/ath11k/mac.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. + * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef ATH11K_MAC_H diff --git a/drivers/net/wireless/ath/ath11k/mhi.c b/drivers/net/wireless/ath/ath11k/mhi.c index afeabd6ecc675d..6835c14b82cc95 100644 --- a/drivers/net/wireless/ath/ath11k/mhi.c +++ b/drivers/net/wireless/ath/ath11k/mhi.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2020 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022, Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include diff --git a/drivers/net/wireless/ath/ath11k/mhi.h b/drivers/net/wireless/ath/ath11k/mhi.h index 8d9f852da69527..f81fba2644a4c7 100644 --- a/drivers/net/wireless/ath/ath11k/mhi.h +++ b/drivers/net/wireless/ath/ath11k/mhi.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2020 The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef _ATH11K_MHI_H #define _ATH11K_MHI_H diff --git a/drivers/net/wireless/ath/ath11k/pcic.c b/drivers/net/wireless/ath/ath11k/pcic.c index 16d1e332193f08..15e2ceb22a44d7 100644 --- a/drivers/net/wireless/ath/ath11k/pcic.c +++ b/drivers/net/wireless/ath/ath11k/pcic.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2019-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022, Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include "core.h" @@ -460,8 +460,6 @@ void ath11k_pcic_ext_irq_enable(struct ath11k_base *ab) { int i; - set_bit(ATH11K_FLAG_EXT_IRQ_ENABLED, &ab->dev_flags); - for (i = 0; i < ATH11K_EXT_IRQ_GRP_NUM_MAX; i++) { struct ath11k_ext_irq_grp *irq_grp = &ab->ext_irq_grp[i]; @@ -471,6 +469,8 @@ void ath11k_pcic_ext_irq_enable(struct ath11k_base *ab) } ath11k_pcic_ext_grp_enable(irq_grp); } + + set_bit(ATH11K_FLAG_EXT_IRQ_ENABLED, &ab->dev_flags); } EXPORT_SYMBOL(ath11k_pcic_ext_irq_enable); diff --git a/drivers/net/wireless/ath/ath11k/peer.c b/drivers/net/wireless/ath/ath11k/peer.c index 1c79a932d17f3b..6d0126c3930185 100644 --- a/drivers/net/wireless/ath/ath11k/peer.c +++ b/drivers/net/wireless/ath/ath11k/peer.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include "core.h" diff --git a/drivers/net/wireless/ath/ath11k/peer.h b/drivers/net/wireless/ath/ath11k/peer.h index 9bd385d0a38c97..3ad2f3355b14fd 100644 --- a/drivers/net/wireless/ath/ath11k/peer.h +++ b/drivers/net/wireless/ath/ath11k/peer.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef ATH11K_PEER_H diff --git a/drivers/net/wireless/ath/ath11k/qmi.c b/drivers/net/wireless/ath/ath11k/qmi.c index c270dc46d50656..2c7cab62b9bb57 100644 --- a/drivers/net/wireless/ath/ath11k/qmi.c +++ b/drivers/net/wireless/ath/ath11k/qmi.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. - * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include diff --git a/drivers/net/wireless/ath/ath11k/qmi.h b/drivers/net/wireless/ath/ath11k/qmi.h index d477e2be814b10..7e06d100af5759 100644 --- a/drivers/net/wireless/ath/ath11k/qmi.h +++ b/drivers/net/wireless/ath/ath11k/qmi.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. - * Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef ATH11K_QMI_H diff --git a/drivers/net/wireless/ath/ath11k/reg.c b/drivers/net/wireless/ath/ath11k/reg.c index 3c7debae800a64..b4fd4d2107c71f 100644 --- a/drivers/net/wireless/ath/ath11k/reg.c +++ b/drivers/net/wireless/ath/ath11k/reg.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include diff --git a/drivers/net/wireless/ath/ath11k/reg.h b/drivers/net/wireless/ath/ath11k/reg.h index 84daa6543b6a32..f28902f85e4196 100644 --- a/drivers/net/wireless/ath/ath11k/reg.h +++ b/drivers/net/wireless/ath/ath11k/reg.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2019 The Linux Foundation. All rights reserved. + * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef ATH11K_REG_H diff --git a/drivers/net/wireless/ath/ath11k/rx_desc.h b/drivers/net/wireless/ath/ath11k/rx_desc.h index 786d5f36f5e547..2da6da72727892 100644 --- a/drivers/net/wireless/ath/ath11k/rx_desc.h +++ b/drivers/net/wireless/ath/ath11k/rx_desc.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef ATH11K_RX_DESC_H #define ATH11K_RX_DESC_H diff --git a/drivers/net/wireless/ath/ath11k/spectral.c b/drivers/net/wireless/ath/ath11k/spectral.c index 0b7b7122cc056c..79e091134515b4 100644 --- a/drivers/net/wireless/ath/ath11k/spectral.c +++ b/drivers/net/wireless/ath/ath11k/spectral.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2019-2020 The Linux Foundation. All rights reserved. + * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #include diff --git a/drivers/net/wireless/ath/ath11k/spectral.h b/drivers/net/wireless/ath/ath11k/spectral.h index 96bfa16e18e96d..789cff7c64a72d 100644 --- a/drivers/net/wireless/ath/ath11k/spectral.h +++ b/drivers/net/wireless/ath/ath11k/spectral.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2019-2020 The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef ATH11K_SPECTRAL_H diff --git a/drivers/net/wireless/ath/ath11k/thermal.c b/drivers/net/wireless/ath/ath11k/thermal.c index c9b012f97ba54c..c29b11ab5bfa45 100644 --- a/drivers/net/wireless/ath/ath11k/thermal.c +++ b/drivers/net/wireless/ath/ath11k/thermal.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2020 The Linux Foundation. All rights reserved. + * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include diff --git a/drivers/net/wireless/ath/ath11k/thermal.h b/drivers/net/wireless/ath/ath11k/thermal.h index 83cb6768673374..cdaf4e01d92eab 100644 --- a/drivers/net/wireless/ath/ath11k/thermal.h +++ b/drivers/net/wireless/ath/ath11k/thermal.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2020 The Linux Foundation. All rights reserved. + * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef _ATH11K_THERMAL_ diff --git a/drivers/net/wireless/ath/ath11k/trace.h b/drivers/net/wireless/ath/ath11k/trace.h index 9535745fe026c2..235ab8ea715fed 100644 --- a/drivers/net/wireless/ath/ath11k/trace.h +++ b/drivers/net/wireless/ath/ath11k/trace.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2019 The Linux Foundation. All rights reserved. + * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #if !defined(_TRACE_H_) || defined(TRACE_HEADER_MULTI_READ) diff --git a/drivers/net/wireless/ath/ath11k/wmi.c b/drivers/net/wireless/ath/ath11k/wmi.c index 2845b4313d3abc..8a65fa04b48d98 100644 --- a/drivers/net/wireless/ath/ath11k/wmi.c +++ b/drivers/net/wireless/ath/ath11k/wmi.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. - * Copyright (c) 2021, 2023 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include #include diff --git a/drivers/net/wireless/ath/ath11k/wmi.h b/drivers/net/wireless/ath/ath11k/wmi.h index 100bb816b59230..ff0a9a92beeb03 100644 --- a/drivers/net/wireless/ath/ath11k/wmi.h +++ b/drivers/net/wireless/ath/ath11k/wmi.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. - * Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef ATH11K_WMI_H @@ -1096,25 +1096,27 @@ enum wmi_tlv_vdev_param { }; enum wmi_tlv_peer_flags { - WMI_TLV_PEER_AUTH = 0x00000001, - WMI_TLV_PEER_QOS = 0x00000002, - WMI_TLV_PEER_NEED_PTK_4_WAY = 0x00000004, - WMI_TLV_PEER_NEED_GTK_2_WAY = 0x00000010, - WMI_TLV_PEER_APSD = 0x00000800, - WMI_TLV_PEER_HT = 0x00001000, - WMI_TLV_PEER_40MHZ = 0x00002000, - WMI_TLV_PEER_STBC = 0x00008000, - WMI_TLV_PEER_LDPC = 0x00010000, - WMI_TLV_PEER_DYN_MIMOPS = 0x00020000, - WMI_TLV_PEER_STATIC_MIMOPS = 0x00040000, - WMI_TLV_PEER_SPATIAL_MUX = 0x00200000, - WMI_TLV_PEER_VHT = 0x02000000, - WMI_TLV_PEER_80MHZ = 0x04000000, - WMI_TLV_PEER_PMF = 0x08000000, + WMI_PEER_AUTH = 0x00000001, + WMI_PEER_QOS = 0x00000002, + WMI_PEER_NEED_PTK_4_WAY = 0x00000004, + WMI_PEER_NEED_GTK_2_WAY = 0x00000010, + WMI_PEER_HE = 0x00000400, + WMI_PEER_APSD = 0x00000800, + WMI_PEER_HT = 0x00001000, + WMI_PEER_40MHZ = 0x00002000, + WMI_PEER_STBC = 0x00008000, + WMI_PEER_LDPC = 0x00010000, + WMI_PEER_DYN_MIMOPS = 0x00020000, + WMI_PEER_STATIC_MIMOPS = 0x00040000, + WMI_PEER_SPATIAL_MUX = 0x00200000, + WMI_PEER_TWT_REQ = 0x00400000, + WMI_PEER_TWT_RESP = 0x00800000, + WMI_PEER_VHT = 0x02000000, + WMI_PEER_80MHZ = 0x04000000, + WMI_PEER_PMF = 0x08000000, WMI_PEER_IS_P2P_CAPABLE = 0x20000000, WMI_PEER_160MHZ = 0x40000000, WMI_PEER_SAFEMODE_EN = 0x80000000, - }; /** Enum list of TLV Tags for each parameter structure type. */ @@ -2580,7 +2582,6 @@ struct wmi_service_available_event { struct ath11k_pdev_wmi { struct ath11k_wmi_base *wmi_ab; enum ath11k_htc_ep_id eid; - const struct wmi_peer_flags_map *peer_flags; u32 rx_decap_mode; wait_queue_head_t tx_ce_desc_wq; }; @@ -4062,31 +4063,6 @@ struct wmi_unit_test_cmd { #define MAX_SUPPORTED_RATES 128 -#define WMI_PEER_AUTH 0x00000001 -#define WMI_PEER_QOS 0x00000002 -#define WMI_PEER_NEED_PTK_4_WAY 0x00000004 -#define WMI_PEER_NEED_GTK_2_WAY 0x00000010 -#define WMI_PEER_HE 0x00000400 -#define WMI_PEER_APSD 0x00000800 -#define WMI_PEER_HT 0x00001000 -#define WMI_PEER_40MHZ 0x00002000 -#define WMI_PEER_STBC 0x00008000 -#define WMI_PEER_LDPC 0x00010000 -#define WMI_PEER_DYN_MIMOPS 0x00020000 -#define WMI_PEER_STATIC_MIMOPS 0x00040000 -#define WMI_PEER_SPATIAL_MUX 0x00200000 -#define WMI_PEER_TWT_REQ 0x00400000 -#define WMI_PEER_TWT_RESP 0x00800000 -#define WMI_PEER_VHT 0x02000000 -#define WMI_PEER_80MHZ 0x04000000 -#define WMI_PEER_PMF 0x08000000 -/* TODO: Place holder for WLAN_PEER_F_PS_PRESEND_REQUIRED = 0x10000000. - * Need to be cleaned up - */ -#define WMI_PEER_IS_P2P_CAPABLE 0x20000000 -#define WMI_PEER_160MHZ 0x40000000 -#define WMI_PEER_SAFEMODE_EN 0x80000000 - struct beacon_tmpl_params { u8 vdev_id; u32 tim_ie_offset; @@ -5754,7 +5730,6 @@ struct ath11k_wmi_base { struct completion unified_ready; DECLARE_BITMAP(svc_map, WMI_MAX_EXT2_SERVICE); wait_queue_head_t tx_credits_wq; - const struct wmi_peer_flags_map *peer_flags; u32 num_mem_chunks; u32 rx_decap_mode; struct wmi_host_mem_chunk mem_chunks[WMI_MAX_MEM_REQS]; diff --git a/drivers/net/wireless/ath/ath11k/wow.h b/drivers/net/wireless/ath/ath11k/wow.h index 553ba850d910b5..c85811e3f42b2d 100644 --- a/drivers/net/wireless/ath/ath11k/wow.h +++ b/drivers/net/wireless/ath/ath11k/wow.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2020 The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef _WOW_H_ diff --git a/drivers/net/wireless/ath/ath12k/Kconfig b/drivers/net/wireless/ath/ath12k/Kconfig index 4f9c514c13e7f0..e135d2b1b61d66 100644 --- a/drivers/net/wireless/ath/ath12k/Kconfig +++ b/drivers/net/wireless/ath/ath12k/Kconfig @@ -2,7 +2,7 @@ config ATH12K tristate "Qualcomm Technologies Wi-Fi 7 support (ath12k)" depends on MAC80211 && HAS_DMA && PCI - depends on CRYPTO_MICHAEL_MIC + select CRYPTO_MICHAEL_MIC select QCOM_QMI_HELPERS select MHI_BUS select QRTR diff --git a/drivers/net/wireless/ath/ath12k/core.c b/drivers/net/wireless/ath/ath12k/core.c index b936760b51408d..6c01b282fcd330 100644 --- a/drivers/net/wireless/ath/ath12k/core.c +++ b/drivers/net/wireless/ath/ath12k/core.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include @@ -698,13 +698,15 @@ int ath12k_core_qmi_firmware_ready(struct ath12k_base *ab) ret = ath12k_core_rfkill_config(ab); if (ret && ret != -EOPNOTSUPP) { ath12k_err(ab, "failed to config rfkill: %d\n", ret); - goto err_core_stop; + goto err_core_pdev_destroy; } mutex_unlock(&ab->core_lock); return 0; +err_core_pdev_destroy: + ath12k_core_pdev_destroy(ab); err_core_stop: ath12k_core_stop(ab); ath12k_mac_destroy(ab); diff --git a/drivers/net/wireless/ath/ath12k/core.h b/drivers/net/wireless/ath/ath12k/core.h index 68c42ca44fcb5f..8458dc292821a3 100644 --- a/drivers/net/wireless/ath/ath12k/core.h +++ b/drivers/net/wireless/ath/ath12k/core.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef ATH12K_CORE_H @@ -199,6 +199,8 @@ enum ath12k_dev_flags { ATH12K_FLAG_REGISTERED, ATH12K_FLAG_QMI_FAIL, ATH12K_FLAG_HTC_SUSPEND_COMPLETE, + ATH12K_FLAG_CE_IRQ_ENABLED, + ATH12K_FLAG_EXT_IRQ_ENABLED, }; enum ath12k_monitor_flags { @@ -467,7 +469,6 @@ struct ath12k { struct ath12k_base *ab; struct ath12k_pdev *pdev; struct ieee80211_hw *hw; - struct ieee80211_ops *ops; struct ath12k_wmi_pdev *wmi; struct ath12k_pdev_dp dp; u8 mac_addr[ETH_ALEN]; diff --git a/drivers/net/wireless/ath/ath12k/dbring.c b/drivers/net/wireless/ath/ath12k/dbring.c index 8fbf868e6f7ec0..788160c84c6868 100644 --- a/drivers/net/wireless/ath/ath12k/dbring.c +++ b/drivers/net/wireless/ath/ath12k/dbring.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2019-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include "core.h" diff --git a/drivers/net/wireless/ath/ath12k/debug.c b/drivers/net/wireless/ath/ath12k/debug.c index 45d33279e665de..fe5a732ba9ec94 100644 --- a/drivers/net/wireless/ath/ath12k/debug.c +++ b/drivers/net/wireless/ath/ath12k/debug.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include diff --git a/drivers/net/wireless/ath/ath12k/dp.c b/drivers/net/wireless/ath/ath12k/dp.c index 6893466f61f047..a6f81f2f97ef45 100644 --- a/drivers/net/wireless/ath/ath12k/dp.c +++ b/drivers/net/wireless/ath/ath12k/dp.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include @@ -961,9 +961,7 @@ int ath12k_dp_service_srng(struct ath12k_base *ab, struct ath12k_dp *dp = &ab->dp; struct dp_rxdma_ring *rx_ring = &dp->rx_refill_buf_ring; - ath12k_dp_rx_bufs_replenish(ab, 0, rx_ring, 0, - ab->hw_params->hal_params->rx_buf_rbm, - true); + ath12k_dp_rx_bufs_replenish(ab, rx_ring, 0); } /* TODO: Implement handler for other interrupts */ diff --git a/drivers/net/wireless/ath/ath12k/dp.h b/drivers/net/wireless/ath/ath12k/dp.h index 61f765432516bb..1df3cdd461400a 100644 --- a/drivers/net/wireless/ath/ath12k/dp.h +++ b/drivers/net/wireless/ath/ath12k/dp.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef ATH12K_DP_H @@ -31,7 +31,7 @@ struct dp_srng { u32 ring_id; }; -struct dp_rxdma_ring { +struct dp_rxdma_mon_ring { struct dp_srng refill_buf_ring; struct idr bufs_idr; /* Protects bufs_idr */ @@ -39,6 +39,11 @@ struct dp_rxdma_ring { int bufs_max; }; +struct dp_rxdma_ring { + struct dp_srng refill_buf_ring; + int bufs_max; +}; + #define ATH12K_TX_COMPL_NEXT(x) (((x) + 1) % DP_TX_COMP_RING_SIZE) struct dp_tx_ring { @@ -353,8 +358,8 @@ struct ath12k_dp { struct dp_rxdma_ring rx_refill_buf_ring; struct dp_srng rx_mac_buf_ring[MAX_RXDMA_PER_PDEV]; struct dp_srng rxdma_err_dst_ring[MAX_RXDMA_PER_PDEV]; - struct dp_rxdma_ring rxdma_mon_buf_ring; - struct dp_rxdma_ring tx_mon_buf_ring; + struct dp_rxdma_mon_ring rxdma_mon_buf_ring; + struct dp_rxdma_mon_ring tx_mon_buf_ring; struct ath12k_reo_q_addr_lut reoq_lut; }; diff --git a/drivers/net/wireless/ath/ath12k/dp_mon.c b/drivers/net/wireless/ath/ath12k/dp_mon.c index f44bc5494ce735..be4b39f5fa8001 100644 --- a/drivers/net/wireless/ath/ath12k/dp_mon.c +++ b/drivers/net/wireless/ath/ath12k/dp_mon.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2019-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include "dp_mon.h" @@ -797,7 +797,7 @@ ath12k_dp_mon_rx_parse_status_tlv(struct ath12k_base *ab, /* TODO: add msdu start parsing logic */ break; case HAL_MON_BUF_ADDR: { - struct dp_rxdma_ring *buf_ring = &ab->dp.rxdma_mon_buf_ring; + struct dp_rxdma_mon_ring *buf_ring = &ab->dp.rxdma_mon_buf_ring; struct dp_mon_packet_info *packet_info = (struct dp_mon_packet_info *)tlv_data; int buf_id = u32_get_bits(packet_info->cookie, @@ -1091,7 +1091,7 @@ static void ath12k_dp_mon_rx_deliver_msdu(struct ath12k *ar, struct napi_struct spin_unlock_bh(&ar->ab->base_lock); ath12k_dbg(ar->ab, ATH12K_DBG_DATA, - "rx skb %pK len %u peer %pM %u %s %s%s%s%s%s%s%s %srate_idx %u vht_nss %u freq %u band %u flag 0x%x fcs-err %i mic-err %i amsdu-more %i\n", + "rx skb %pK len %u peer %pM %u %s %s%s%s%s%s%s%s%s %srate_idx %u vht_nss %u freq %u band %u flag 0x%x fcs-err %i mic-err %i amsdu-more %i\n", msdu, msdu->len, peer ? peer->addr : NULL, @@ -1104,6 +1104,7 @@ static void ath12k_dp_mon_rx_deliver_msdu(struct ath12k *ar, struct napi_struct (status->bw == RATE_INFO_BW_40) ? "40" : "", (status->bw == RATE_INFO_BW_80) ? "80" : "", (status->bw == RATE_INFO_BW_160) ? "160" : "", + (status->bw == RATE_INFO_BW_320) ? "320" : "", status->enc_flags & RX_ENC_FLAG_SHORT_GI ? "sgi " : "", status->rate_idx, status->nss, @@ -1259,7 +1260,7 @@ ath12k_dp_mon_rx_parse_mon_status(struct ath12k *ar, } int ath12k_dp_mon_buf_replenish(struct ath12k_base *ab, - struct dp_rxdma_ring *buf_ring, + struct dp_rxdma_mon_ring *buf_ring, int req_entries) { struct hal_mon_buf_ring *mon_buf; @@ -1902,7 +1903,7 @@ ath12k_dp_mon_tx_parse_status_tlv(struct ath12k_base *ab, } case HAL_MON_BUF_ADDR: { - struct dp_rxdma_ring *buf_ring = &ab->dp.tx_mon_buf_ring; + struct dp_rxdma_mon_ring *buf_ring = &ab->dp.tx_mon_buf_ring; struct dp_mon_packet_info *packet_info = (struct dp_mon_packet_info *)tlv_data; int buf_id = u32_get_bits(packet_info->cookie, @@ -2067,7 +2068,7 @@ int ath12k_dp_mon_srng_process(struct ath12k *ar, int mac_id, int *budget, struct ath12k_skb_rxcb *rxcb; struct dp_srng *mon_dst_ring; struct hal_srng *srng; - struct dp_rxdma_ring *buf_ring; + struct dp_rxdma_mon_ring *buf_ring; u64 cookie; u32 ppdu_id; int num_buffs_reaped = 0, srng_id, buf_id; @@ -2480,7 +2481,7 @@ int ath12k_dp_mon_rx_process_stats(struct ath12k *ar, int mac_id, struct ath12k_skb_rxcb *rxcb; struct dp_srng *mon_dst_ring; struct hal_srng *srng; - struct dp_rxdma_ring *buf_ring; + struct dp_rxdma_mon_ring *buf_ring; struct ath12k_sta *arsta = NULL; struct ath12k_peer *peer; u64 cookie; diff --git a/drivers/net/wireless/ath/ath12k/dp_mon.h b/drivers/net/wireless/ath/ath12k/dp_mon.h index c18c385798a110..fb9e9c176ce57b 100644 --- a/drivers/net/wireless/ath/ath12k/dp_mon.h +++ b/drivers/net/wireless/ath/ath12k/dp_mon.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2019-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef ATH12K_DP_MON_H @@ -80,7 +80,7 @@ ath12k_dp_mon_rx_parse_mon_status(struct ath12k *ar, int mac_id, struct sk_buff *skb, struct napi_struct *napi); int ath12k_dp_mon_buf_replenish(struct ath12k_base *ab, - struct dp_rxdma_ring *buf_ring, + struct dp_rxdma_mon_ring *buf_ring, int req_entries); int ath12k_dp_mon_srng_process(struct ath12k *ar, int mac_id, int *budget, enum dp_monitor_mode monitor_mode, diff --git a/drivers/net/wireless/ath/ath12k/dp_rx.c b/drivers/net/wireless/ath/ath12k/dp_rx.c index 3543fadac4a576..1ee83f7659293e 100644 --- a/drivers/net/wireless/ath/ath12k/dp_rx.c +++ b/drivers/net/wireless/ath/ath12k/dp_rx.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include @@ -256,22 +256,20 @@ static int ath12k_dp_purge_mon_ring(struct ath12k_base *ab) } /* Returns number of Rx buffers replenished */ -int ath12k_dp_rx_bufs_replenish(struct ath12k_base *ab, int mac_id, +int ath12k_dp_rx_bufs_replenish(struct ath12k_base *ab, struct dp_rxdma_ring *rx_ring, - int req_entries, - enum hal_rx_buf_return_buf_manager mgr, - bool hw_cc) + int req_entries) { struct ath12k_buffer_addr *desc; struct hal_srng *srng; struct sk_buff *skb; int num_free; int num_remain; - int buf_id; u32 cookie; dma_addr_t paddr; struct ath12k_dp *dp = &ab->dp; struct ath12k_rx_desc_info *rx_desc; + enum hal_rx_buf_return_buf_manager mgr = ab->hw_params->hal_params->rx_buf_rbm; req_entries = min(req_entries, rx_ring->bufs_max); @@ -307,42 +305,29 @@ int ath12k_dp_rx_bufs_replenish(struct ath12k_base *ab, int mac_id, if (dma_mapping_error(ab->dev, paddr)) goto fail_free_skb; - if (hw_cc) { - spin_lock_bh(&dp->rx_desc_lock); - - /* Get desc from free list and store in used list - * for cleanup purposes - * - * TODO: pass the removed descs rather than - * add/read to optimize - */ - rx_desc = list_first_entry_or_null(&dp->rx_desc_free_list, - struct ath12k_rx_desc_info, - list); - if (!rx_desc) { - spin_unlock_bh(&dp->rx_desc_lock); - goto fail_dma_unmap; - } - - rx_desc->skb = skb; - cookie = rx_desc->cookie; - list_del(&rx_desc->list); - list_add_tail(&rx_desc->list, &dp->rx_desc_used_list); + spin_lock_bh(&dp->rx_desc_lock); + /* Get desc from free list and store in used list + * for cleanup purposes + * + * TODO: pass the removed descs rather than + * add/read to optimize + */ + rx_desc = list_first_entry_or_null(&dp->rx_desc_free_list, + struct ath12k_rx_desc_info, + list); + if (!rx_desc) { spin_unlock_bh(&dp->rx_desc_lock); - } else { - spin_lock_bh(&rx_ring->idr_lock); - buf_id = idr_alloc(&rx_ring->bufs_idr, skb, 0, - rx_ring->bufs_max * 3, GFP_ATOMIC); - spin_unlock_bh(&rx_ring->idr_lock); - if (buf_id < 0) - goto fail_dma_unmap; - cookie = u32_encode_bits(mac_id, - DP_RXDMA_BUF_COOKIE_PDEV_ID) | - u32_encode_bits(buf_id, - DP_RXDMA_BUF_COOKIE_BUF_ID); + goto fail_dma_unmap; } + rx_desc->skb = skb; + cookie = rx_desc->cookie; + list_del(&rx_desc->list); + list_add_tail(&rx_desc->list, &dp->rx_desc_used_list); + + spin_unlock_bh(&dp->rx_desc_lock); + desc = ath12k_hal_srng_src_get_next_entry(ab, srng); if (!desc) goto fail_buf_unassign; @@ -361,17 +346,11 @@ int ath12k_dp_rx_bufs_replenish(struct ath12k_base *ab, int mac_id, return req_entries - num_remain; fail_buf_unassign: - if (hw_cc) { - spin_lock_bh(&dp->rx_desc_lock); - list_del(&rx_desc->list); - list_add_tail(&rx_desc->list, &dp->rx_desc_free_list); - rx_desc->skb = NULL; - spin_unlock_bh(&dp->rx_desc_lock); - } else { - spin_lock_bh(&rx_ring->idr_lock); - idr_remove(&rx_ring->bufs_idr, buf_id); - spin_unlock_bh(&rx_ring->idr_lock); - } + spin_lock_bh(&dp->rx_desc_lock); + list_del(&rx_desc->list); + list_add_tail(&rx_desc->list, &dp->rx_desc_free_list); + rx_desc->skb = NULL; + spin_unlock_bh(&dp->rx_desc_lock); fail_dma_unmap: dma_unmap_single(ab->dev, paddr, skb->len + skb_tailroom(skb), DMA_FROM_DEVICE); @@ -385,8 +364,8 @@ int ath12k_dp_rx_bufs_replenish(struct ath12k_base *ab, int mac_id, return req_entries - num_remain; } -static int ath12k_dp_rxdma_buf_ring_free(struct ath12k_base *ab, - struct dp_rxdma_ring *rx_ring) +static int ath12k_dp_rxdma_mon_buf_ring_free(struct ath12k_base *ab, + struct dp_rxdma_mon_ring *rx_ring) { struct sk_buff *skb; int buf_id; @@ -411,46 +390,49 @@ static int ath12k_dp_rxdma_buf_ring_free(struct ath12k_base *ab, static int ath12k_dp_rxdma_buf_free(struct ath12k_base *ab) { struct ath12k_dp *dp = &ab->dp; - struct dp_rxdma_ring *rx_ring = &dp->rx_refill_buf_ring; - ath12k_dp_rxdma_buf_ring_free(ab, rx_ring); + ath12k_dp_rxdma_mon_buf_ring_free(ab, &dp->rxdma_mon_buf_ring); + + ath12k_dp_rxdma_mon_buf_ring_free(ab, &dp->tx_mon_buf_ring); + + return 0; +} - rx_ring = &dp->rxdma_mon_buf_ring; - ath12k_dp_rxdma_buf_ring_free(ab, rx_ring); +static int ath12k_dp_rxdma_mon_ring_buf_setup(struct ath12k_base *ab, + struct dp_rxdma_mon_ring *rx_ring, + u32 ringtype) +{ + int num_entries; - rx_ring = &dp->tx_mon_buf_ring; - ath12k_dp_rxdma_buf_ring_free(ab, rx_ring); + num_entries = rx_ring->refill_buf_ring.size / + ath12k_hal_srng_get_entrysize(ab, ringtype); + + rx_ring->bufs_max = num_entries; + ath12k_dp_mon_buf_replenish(ab, rx_ring, num_entries); return 0; } static int ath12k_dp_rxdma_ring_buf_setup(struct ath12k_base *ab, - struct dp_rxdma_ring *rx_ring, - u32 ringtype) + struct dp_rxdma_ring *rx_ring) { int num_entries; num_entries = rx_ring->refill_buf_ring.size / - ath12k_hal_srng_get_entrysize(ab, ringtype); + ath12k_hal_srng_get_entrysize(ab, HAL_RXDMA_BUF); rx_ring->bufs_max = num_entries; - if ((ringtype == HAL_RXDMA_MONITOR_BUF) || (ringtype == HAL_TX_MONITOR_BUF)) - ath12k_dp_mon_buf_replenish(ab, rx_ring, num_entries); - else - ath12k_dp_rx_bufs_replenish(ab, 0, rx_ring, num_entries, - ab->hw_params->hal_params->rx_buf_rbm, - ringtype == HAL_RXDMA_BUF); + ath12k_dp_rx_bufs_replenish(ab, rx_ring, num_entries); + return 0; } static int ath12k_dp_rxdma_buf_setup(struct ath12k_base *ab) { struct ath12k_dp *dp = &ab->dp; - struct dp_rxdma_ring *rx_ring = &dp->rx_refill_buf_ring; int ret; - ret = ath12k_dp_rxdma_ring_buf_setup(ab, rx_ring, - HAL_RXDMA_BUF); + ret = ath12k_dp_rxdma_ring_buf_setup(ab, &dp->rx_refill_buf_ring); if (ret) { ath12k_warn(ab, "failed to setup HAL_RXDMA_BUF\n"); @@ -458,18 +440,18 @@ static int ath12k_dp_rxdma_buf_setup(struct ath12k_base *ab) } if (ab->hw_params->rxdma1_enable) { - rx_ring = &dp->rxdma_mon_buf_ring; - ret = ath12k_dp_rxdma_ring_buf_setup(ab, rx_ring, - HAL_RXDMA_MONITOR_BUF); + ret = ath12k_dp_rxdma_mon_ring_buf_setup(ab, + &dp->rxdma_mon_buf_ring, + HAL_RXDMA_MONITOR_BUF); if (ret) { ath12k_warn(ab, "failed to setup HAL_RXDMA_MONITOR_BUF\n"); return ret; } - rx_ring = &dp->tx_mon_buf_ring; - ret = ath12k_dp_rxdma_ring_buf_setup(ab, rx_ring, - HAL_TX_MONITOR_BUF); + ret = ath12k_dp_rxdma_mon_ring_buf_setup(ab, + &dp->tx_mon_buf_ring, + HAL_TX_MONITOR_BUF); if (ret) { ath12k_warn(ab, "failed to setup HAL_TX_MONITOR_BUF\n"); @@ -1339,9 +1321,6 @@ ath12k_update_per_peer_tx_stats(struct ath12k *ar, u8 tid = HTT_PPDU_STATS_NON_QOS_TID; bool is_ampdu = false; - if (!usr_stats) - return; - if (!(usr_stats->tlv_flags & BIT(HTT_PPDU_STATS_TAG_USR_RATE))) return; @@ -2438,7 +2417,7 @@ static void ath12k_dp_rx_deliver_msdu(struct ath12k *ar, struct napi_struct *nap spin_unlock_bh(&ab->base_lock); ath12k_dbg(ab, ATH12K_DBG_DATA, - "rx skb %pK len %u peer %pM %d %s sn %u %s%s%s%s%s%s%s%s rate_idx %u vht_nss %u freq %u band %u flag 0x%x fcs-err %i mic-err %i amsdu-more %i\n", + "rx skb %pK len %u peer %pM %d %s sn %u %s%s%s%s%s%s%s%s%s rate_idx %u vht_nss %u freq %u band %u flag 0x%x fcs-err %i mic-err %i amsdu-more %i\n", msdu, msdu->len, peer ? peer->addr : NULL, @@ -2452,6 +2431,7 @@ static void ath12k_dp_rx_deliver_msdu(struct ath12k *ar, struct napi_struct *nap (status->bw == RATE_INFO_BW_40) ? "40" : "", (status->bw == RATE_INFO_BW_80) ? "80" : "", (status->bw == RATE_INFO_BW_160) ? "160" : "", + (status->bw == RATE_INFO_BW_320) ? "320" : "", status->enc_flags & RX_ENC_FLAG_SHORT_GI ? "sgi " : "", status->rate_idx, status->nss, @@ -2714,9 +2694,7 @@ int ath12k_dp_rx_process(struct ath12k_base *ab, int ring_id, if (!total_msdu_reaped) goto exit; - /* TODO: Move to implicit BM? */ - ath12k_dp_rx_bufs_replenish(ab, 0, rx_ring, num_buffs_reaped, - ab->hw_params->hal_params->rx_buf_rbm, true); + ath12k_dp_rx_bufs_replenish(ab, rx_ring, num_buffs_reaped); ath12k_dp_rx_process_received_packets(ab, napi, &msdu_list, ring_id); @@ -3494,8 +3472,7 @@ int ath12k_dp_rx_process_err(struct ath12k_base *ab, struct napi_struct *napi, rx_ring = &dp->rx_refill_buf_ring; - ath12k_dp_rx_bufs_replenish(ab, 0, rx_ring, tot_n_bufs_reaped, - ab->hw_params->hal_params->rx_buf_rbm, true); + ath12k_dp_rx_bufs_replenish(ab, rx_ring, tot_n_bufs_reaped); return tot_n_bufs_reaped; } @@ -3808,8 +3785,7 @@ int ath12k_dp_rx_process_wbm_err(struct ath12k_base *ab, if (!num_buffs_reaped) goto done; - ath12k_dp_rx_bufs_replenish(ab, 0, rx_ring, num_buffs_reaped, - ab->hw_params->hal_params->rx_buf_rbm, true); + ath12k_dp_rx_bufs_replenish(ab, rx_ring, num_buffs_reaped); rcu_read_lock(); for (i = 0; i < ab->num_radios; i++) { @@ -4090,9 +4066,6 @@ int ath12k_dp_rx_alloc(struct ath12k_base *ab) struct ath12k_dp *dp = &ab->dp; int i, ret; - idr_init(&dp->rx_refill_buf_ring.bufs_idr); - spin_lock_init(&dp->rx_refill_buf_ring.idr_lock); - idr_init(&dp->rxdma_mon_buf_ring.bufs_idr); spin_lock_init(&dp->rxdma_mon_buf_ring.idr_lock); diff --git a/drivers/net/wireless/ath/ath12k/dp_rx.h b/drivers/net/wireless/ath/ath12k/dp_rx.h index c955b5c859d147..05b3d5581dbe2b 100644 --- a/drivers/net/wireless/ath/ath12k/dp_rx.h +++ b/drivers/net/wireless/ath/ath12k/dp_rx.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef ATH12K_DP_RX_H #define ATH12K_DP_RX_H @@ -116,11 +116,9 @@ int ath12k_dp_rx_process_err(struct ath12k_base *ab, struct napi_struct *napi, int ath12k_dp_rx_process(struct ath12k_base *ab, int mac_id, struct napi_struct *napi, int budget); -int ath12k_dp_rx_bufs_replenish(struct ath12k_base *ab, int mac_id, +int ath12k_dp_rx_bufs_replenish(struct ath12k_base *ab, struct dp_rxdma_ring *rx_ring, - int req_entries, - enum hal_rx_buf_return_buf_manager mgr, - bool hw_cc); + int req_entries); int ath12k_dp_rx_pdev_mon_attach(struct ath12k *ar); int ath12k_dp_rx_peer_frag_setup(struct ath12k *ar, const u8 *peer_mac, int vdev_id); diff --git a/drivers/net/wireless/ath/ath12k/dp_tx.c b/drivers/net/wireless/ath/ath12k/dp_tx.c index 492ca6ce67140a..62f9cdbb811c02 100644 --- a/drivers/net/wireless/ath/ath12k/dp_tx.c +++ b/drivers/net/wireless/ath/ath12k/dp_tx.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include "core.h" diff --git a/drivers/net/wireless/ath/ath12k/hal.c b/drivers/net/wireless/ath/ath12k/hal.c index eca86fc25a6084..a489369d806870 100644 --- a/drivers/net/wireless/ath/ath12k/hal.c +++ b/drivers/net/wireless/ath/ath12k/hal.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include #include "hal_tx.h" @@ -889,8 +889,8 @@ static u8 *ath12k_hw_wcn7850_rx_desc_mpdu_start_addr2(struct hal_rx_desc *desc) static bool ath12k_hw_wcn7850_rx_desc_is_da_mcbc(struct hal_rx_desc *desc) { - return __le16_to_cpu(desc->u.wcn7850.msdu_end.info5) & - RX_MSDU_END_INFO5_DA_IS_MCBC; + return __le32_to_cpu(desc->u.wcn7850.msdu_end.info13) & + RX_MSDU_END_INFO13_MCAST_BCAST; } static void ath12k_hw_wcn7850_rx_desc_get_dot11_hdr(struct hal_rx_desc *desc, diff --git a/drivers/net/wireless/ath/ath12k/hal.h b/drivers/net/wireless/ath/ath12k/hal.h index 66035a787c728d..fc47e7e6b498a1 100644 --- a/drivers/net/wireless/ath/ath12k/hal.h +++ b/drivers/net/wireless/ath/ath12k/hal.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef ATH12K_HAL_H diff --git a/drivers/net/wireless/ath/ath12k/hal_rx.c b/drivers/net/wireless/ath/ath12k/hal_rx.c index f6afbd8196bf53..4f25eb9f774538 100644 --- a/drivers/net/wireless/ath/ath12k/hal_rx.c +++ b/drivers/net/wireless/ath/ath12k/hal_rx.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include "debug.h" diff --git a/drivers/net/wireless/ath/ath12k/hal_rx.h b/drivers/net/wireless/ath/ath12k/hal_rx.h index fcfb6c81904736..095216eabc01dc 100644 --- a/drivers/net/wireless/ath/ath12k/hal_rx.h +++ b/drivers/net/wireless/ath/ath12k/hal_rx.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef ATH12K_HAL_RX_H @@ -61,6 +61,7 @@ enum hal_rx_bw { HAL_RX_BW_40MHZ, HAL_RX_BW_80MHZ, HAL_RX_BW_160MHZ, + HAL_RX_BW_320MHZ, HAL_RX_BW_MAX, }; diff --git a/drivers/net/wireless/ath/ath12k/hif.h b/drivers/net/wireless/ath/ath12k/hif.h index 4095fd82b1b3ff..c653ca1f59b22d 100644 --- a/drivers/net/wireless/ath/ath12k/hif.h +++ b/drivers/net/wireless/ath/ath12k/hif.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2019-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef ATH12K_HIF_H diff --git a/drivers/net/wireless/ath/ath12k/hw.c b/drivers/net/wireless/ath/ath12k/hw.c index 2245fb510ba2ca..de60d988d8608f 100644 --- a/drivers/net/wireless/ath/ath12k/hw.c +++ b/drivers/net/wireless/ath/ath12k/hw.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include @@ -949,7 +949,8 @@ static const struct ath12k_hw_params ath12k_hw_params[] = { .rx_mac_buf_ring = true, .vdev_start_delay = true, - .interface_modes = BIT(NL80211_IFTYPE_STATION), + .interface_modes = BIT(NL80211_IFTYPE_STATION) | + BIT(NL80211_IFTYPE_AP), .supports_monitor = false, .idle_ps = true, diff --git a/drivers/net/wireless/ath/ath12k/hw.h b/drivers/net/wireless/ath/ath12k/hw.h index 2d6427cf41a4d1..d2622bfef94226 100644 --- a/drivers/net/wireless/ath/ath12k/hw.h +++ b/drivers/net/wireless/ath/ath12k/hw.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef ATH12K_HW_H diff --git a/drivers/net/wireless/ath/ath12k/mac.c b/drivers/net/wireless/ath/ath12k/mac.c index fc0d14ea328e61..88cec54c6c2e64 100644 --- a/drivers/net/wireless/ath/ath12k/mac.c +++ b/drivers/net/wireless/ath/ath12k/mac.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include @@ -343,6 +343,9 @@ ath12k_mac_bw_to_mac80211_bw(enum ath12k_supported_bw bw) case ATH12K_BW_160: ret = RATE_INFO_BW_160; break; + case ATH12K_BW_320: + ret = RATE_INFO_BW_320; + break; } return ret; @@ -359,6 +362,8 @@ enum ath12k_supported_bw ath12k_mac_mac80211_bw_to_ath12k_bw(enum rate_info_bw b return ATH12K_BW_80; case RATE_INFO_BW_160: return ATH12K_BW_160; + case RATE_INFO_BW_320: + return ATH12K_BW_320; default: return ATH12K_BW_20; } @@ -3726,6 +3731,9 @@ static u32 ath12k_mac_ieee80211_sta_bw_to_wmi(struct ath12k *ar, case IEEE80211_STA_RX_BW_160: bw = WMI_PEER_CHWIDTH_160MHZ; break; + case IEEE80211_STA_RX_BW_320: + bw = WMI_PEER_CHWIDTH_320MHZ; + break; default: ath12k_warn(ar->ab, "Invalid bandwidth %d in rc update for %pM\n", sta->deflink.bandwidth, sta->addr); @@ -4987,7 +4995,7 @@ static void ath12k_mac_op_tx(struct ieee80211_hw *hw, if (ret) { ath12k_warn(ar->ab, "failed to queue management frame %d\n", ret); - ieee80211_free_txskb(ar->hw, skb); + ieee80211_free_txskb(hw, skb); } return; } @@ -4995,7 +5003,7 @@ static void ath12k_mac_op_tx(struct ieee80211_hw *hw, ret = ath12k_dp_tx(ar, arvif, skb); if (ret) { ath12k_warn(ar->ab, "failed to transmit frame %d\n", ret); - ieee80211_free_txskb(ar->hw, skb); + ieee80211_free_txskb(hw, skb); } } @@ -5596,7 +5604,7 @@ static int ath12k_mac_op_add_interface(struct ieee80211_hw *hw, goto err_peer_del; param_id = WMI_VDEV_PARAM_RTS_THRESHOLD; - param_value = ar->hw->wiphy->rts_threshold; + param_value = hw->wiphy->rts_threshold; ret = ath12k_wmi_vdev_set_param_cmd(ar, arvif->vdev_id, param_id, param_value); if (ret) { @@ -6258,10 +6266,11 @@ ath12k_mac_update_active_vif_chan(struct ath12k *ar, struct ieee80211_chanctx_conf *ctx) { struct ath12k_mac_change_chanctx_arg arg = { .ctx = ctx }; + struct ieee80211_hw *hw = ar->hw; lockdep_assert_held(&ar->conf_mutex); - ieee80211_iterate_active_interfaces_atomic(ar->hw, + ieee80211_iterate_active_interfaces_atomic(hw, IEEE80211_IFACE_ITER_NORMAL, ath12k_mac_change_chanctx_cnt_iter, &arg); @@ -6272,7 +6281,7 @@ ath12k_mac_update_active_vif_chan(struct ath12k *ar, if (!arg.vifs) return; - ieee80211_iterate_active_interfaces_atomic(ar->hw, + ieee80211_iterate_active_interfaces_atomic(hw, IEEE80211_IFACE_ITER_NORMAL, ath12k_mac_change_chanctx_fill_iter, &arg); @@ -6380,8 +6389,8 @@ ath12k_mac_op_assign_vif_chanctx(struct ieee80211_hw *hw, } if (ab->hw_params->vdev_start_delay && - (arvif->vdev_type == WMI_VDEV_TYPE_AP || - arvif->vdev_type == WMI_VDEV_TYPE_MONITOR)) { + arvif->vdev_type != WMI_VDEV_TYPE_AP && + arvif->vdev_type != WMI_VDEV_TYPE_MONITOR) { param.vdev_id = arvif->vdev_id; param.peer_type = WMI_PEER_TYPE_DEFAULT; param.peer_addr = ar->mac_addr; @@ -6836,7 +6845,7 @@ ath12k_mac_op_set_bitrate_mask(struct ieee80211_hw *hw, arvif->vdev_id, ret); return ret; } - ieee80211_iterate_stations_atomic(ar->hw, + ieee80211_iterate_stations_atomic(hw, ath12k_mac_disable_peer_fixed_rate, arvif); } else if (ath12k_mac_bitrate_mask_get_single_nss(ar, band, mask, @@ -6882,14 +6891,14 @@ ath12k_mac_op_set_bitrate_mask(struct ieee80211_hw *hw, return -EINVAL; } - ieee80211_iterate_stations_atomic(ar->hw, + ieee80211_iterate_stations_atomic(hw, ath12k_mac_disable_peer_fixed_rate, arvif); mutex_lock(&ar->conf_mutex); arvif->bitrate_mask = *mask; - ieee80211_iterate_stations_atomic(ar->hw, + ieee80211_iterate_stations_atomic(hw, ath12k_mac_set_bitrate_mask_iter, arvif); @@ -6927,7 +6936,7 @@ ath12k_mac_op_reconfig_complete(struct ieee80211_hw *hw, ath12k_warn(ar->ab, "pdev %d successfully recovered\n", ar->pdev->pdev_id); ar->state = ATH12K_STATE_ON; - ieee80211_wake_queues(ar->hw); + ieee80211_wake_queues(hw); if (ab->is_reset) { recovery_count = atomic_inc_return(&ab->recovery_count); @@ -7151,6 +7160,7 @@ static u32 ath12k_get_phy_id(struct ath12k *ar, u32 band) static int ath12k_mac_setup_channels_rates(struct ath12k *ar, u32 supported_bands) { + struct ieee80211_hw *hw = ar->hw; struct ieee80211_supported_band *band; struct ath12k_wmi_hal_reg_capabilities_ext_arg *reg_cap; void *channels; @@ -7176,7 +7186,7 @@ static int ath12k_mac_setup_channels_rates(struct ath12k *ar, band->channels = channels; band->n_bitrates = ath12k_g_rates_size; band->bitrates = ath12k_g_rates; - ar->hw->wiphy->bands[NL80211_BAND_2GHZ] = band; + hw->wiphy->bands[NL80211_BAND_2GHZ] = band; if (ar->ab->hw_params->single_pdev_only) { phy_id = ath12k_get_phy_id(ar, WMI_HOST_WLAN_2G_CAP); @@ -7203,7 +7213,7 @@ static int ath12k_mac_setup_channels_rates(struct ath12k *ar, band->channels = channels; band->n_bitrates = ath12k_a_rates_size; band->bitrates = ath12k_a_rates; - ar->hw->wiphy->bands[NL80211_BAND_6GHZ] = band; + hw->wiphy->bands[NL80211_BAND_6GHZ] = band; ath12k_mac_update_ch_list(ar, band, reg_cap->low_5ghz_chan, reg_cap->high_5ghz_chan); @@ -7225,7 +7235,7 @@ static int ath12k_mac_setup_channels_rates(struct ath12k *ar, band->channels = channels; band->n_bitrates = ath12k_a_rates_size; band->bitrates = ath12k_a_rates; - ar->hw->wiphy->bands[NL80211_BAND_5GHZ] = band; + hw->wiphy->bands[NL80211_BAND_5GHZ] = band; if (ar->ab->hw_params->single_pdev_only) { phy_id = ath12k_get_phy_id(ar, WMI_HOST_WLAN_5G_CAP); @@ -7244,6 +7254,8 @@ static int ath12k_mac_setup_channels_rates(struct ath12k *ar, static int ath12k_mac_setup_iface_combinations(struct ath12k *ar) { struct ath12k_base *ab = ar->ab; + struct ieee80211_hw *hw = ar->hw; + struct wiphy *wiphy = hw->wiphy; struct ieee80211_iface_combination *combinations; struct ieee80211_iface_limit *limits; int n_limits, max_interfaces; @@ -7294,8 +7306,8 @@ static int ath12k_mac_setup_iface_combinations(struct ath12k *ar) BIT(NL80211_CHAN_WIDTH_40) | BIT(NL80211_CHAN_WIDTH_80); - ar->hw->wiphy->iface_combinations = combinations; - ar->hw->wiphy->n_iface_combinations = 1; + wiphy->iface_combinations = combinations; + wiphy->n_iface_combinations = 1; return 0; } @@ -7339,9 +7351,12 @@ static const struct wiphy_iftype_ext_capab ath12k_iftypes_ext_capa[] = { static void __ath12k_mac_unregister(struct ath12k *ar) { + struct ieee80211_hw *hw = ar->hw; + struct wiphy *wiphy = hw->wiphy; + cancel_work_sync(&ar->regd_update_work); - ieee80211_unregister_hw(ar->hw); + ieee80211_unregister_hw(hw); idr_for_each(&ar->txmgmt_idr, ath12k_mac_tx_mgmt_pending_free, ar); idr_destroy(&ar->txmgmt_idr); @@ -7350,10 +7365,10 @@ static void __ath12k_mac_unregister(struct ath12k *ar) kfree(ar->mac.sbands[NL80211_BAND_5GHZ].channels); kfree(ar->mac.sbands[NL80211_BAND_6GHZ].channels); - kfree(ar->hw->wiphy->iface_combinations[0].limits); - kfree(ar->hw->wiphy->iface_combinations); + kfree(wiphy->iface_combinations[0].limits); + kfree(wiphy->iface_combinations); - SET_IEEE80211_DEV(ar->hw, NULL); + SET_IEEE80211_DEV(hw, NULL); } void ath12k_mac_unregister(struct ath12k_base *ab) @@ -7375,6 +7390,8 @@ void ath12k_mac_unregister(struct ath12k_base *ab) static int __ath12k_mac_register(struct ath12k *ar) { struct ath12k_base *ab = ar->ab; + struct ieee80211_hw *hw = ar->hw; + struct wiphy *wiphy = hw->wiphy; struct ath12k_pdev_cap *cap = &ar->pdev->cap; static const u32 cipher_suites[] = { WLAN_CIPHER_SUITE_TKIP, @@ -7392,9 +7409,9 @@ static int __ath12k_mac_register(struct ath12k *ar) ath12k_pdev_caps_update(ar); - SET_IEEE80211_PERM_ADDR(ar->hw, ar->mac_addr); + SET_IEEE80211_PERM_ADDR(hw, ar->mac_addr); - SET_IEEE80211_DEV(ar->hw, ab->dev); + SET_IEEE80211_DEV(hw, ab->dev); ret = ath12k_mac_setup_channels_rates(ar, cap->supported_bands); @@ -7410,103 +7427,102 @@ static int __ath12k_mac_register(struct ath12k *ar) goto err_free_channels; } - ar->hw->wiphy->available_antennas_rx = cap->rx_chain_mask; - ar->hw->wiphy->available_antennas_tx = cap->tx_chain_mask; + wiphy->available_antennas_rx = cap->rx_chain_mask; + wiphy->available_antennas_tx = cap->tx_chain_mask; - ar->hw->wiphy->interface_modes = ab->hw_params->interface_modes; + wiphy->interface_modes = ab->hw_params->interface_modes; - if (ar->hw->wiphy->bands[NL80211_BAND_2GHZ] && - ar->hw->wiphy->bands[NL80211_BAND_5GHZ] && - ar->hw->wiphy->bands[NL80211_BAND_6GHZ]) - ieee80211_hw_set(ar->hw, SINGLE_SCAN_ON_ALL_BANDS); + if (wiphy->bands[NL80211_BAND_2GHZ] && + wiphy->bands[NL80211_BAND_5GHZ] && + wiphy->bands[NL80211_BAND_6GHZ]) + ieee80211_hw_set(hw, SINGLE_SCAN_ON_ALL_BANDS); - ieee80211_hw_set(ar->hw, SIGNAL_DBM); - ieee80211_hw_set(ar->hw, SUPPORTS_PS); - ieee80211_hw_set(ar->hw, SUPPORTS_DYNAMIC_PS); - ieee80211_hw_set(ar->hw, MFP_CAPABLE); - ieee80211_hw_set(ar->hw, REPORTS_TX_ACK_STATUS); - ieee80211_hw_set(ar->hw, HAS_RATE_CONTROL); - ieee80211_hw_set(ar->hw, AP_LINK_PS); - ieee80211_hw_set(ar->hw, SPECTRUM_MGMT); - ieee80211_hw_set(ar->hw, CONNECTION_MONITOR); - ieee80211_hw_set(ar->hw, SUPPORTS_PER_STA_GTK); - ieee80211_hw_set(ar->hw, CHANCTX_STA_CSA); - ieee80211_hw_set(ar->hw, QUEUE_CONTROL); - ieee80211_hw_set(ar->hw, SUPPORTS_TX_FRAG); - ieee80211_hw_set(ar->hw, REPORTS_LOW_ACK); + ieee80211_hw_set(hw, SIGNAL_DBM); + ieee80211_hw_set(hw, SUPPORTS_PS); + ieee80211_hw_set(hw, SUPPORTS_DYNAMIC_PS); + ieee80211_hw_set(hw, MFP_CAPABLE); + ieee80211_hw_set(hw, REPORTS_TX_ACK_STATUS); + ieee80211_hw_set(hw, HAS_RATE_CONTROL); + ieee80211_hw_set(hw, AP_LINK_PS); + ieee80211_hw_set(hw, SPECTRUM_MGMT); + ieee80211_hw_set(hw, CONNECTION_MONITOR); + ieee80211_hw_set(hw, SUPPORTS_PER_STA_GTK); + ieee80211_hw_set(hw, CHANCTX_STA_CSA); + ieee80211_hw_set(hw, QUEUE_CONTROL); + ieee80211_hw_set(hw, SUPPORTS_TX_FRAG); + ieee80211_hw_set(hw, REPORTS_LOW_ACK); if (ht_cap & WMI_HT_CAP_ENABLED) { - ieee80211_hw_set(ar->hw, AMPDU_AGGREGATION); - ieee80211_hw_set(ar->hw, TX_AMPDU_SETUP_IN_HW); - ieee80211_hw_set(ar->hw, SUPPORTS_REORDERING_BUFFER); - ieee80211_hw_set(ar->hw, SUPPORTS_AMSDU_IN_AMPDU); - ieee80211_hw_set(ar->hw, USES_RSS); + ieee80211_hw_set(hw, AMPDU_AGGREGATION); + ieee80211_hw_set(hw, TX_AMPDU_SETUP_IN_HW); + ieee80211_hw_set(hw, SUPPORTS_REORDERING_BUFFER); + ieee80211_hw_set(hw, SUPPORTS_AMSDU_IN_AMPDU); + ieee80211_hw_set(hw, USES_RSS); } - ar->hw->wiphy->features |= NL80211_FEATURE_STATIC_SMPS; - ar->hw->wiphy->flags |= WIPHY_FLAG_IBSS_RSN; + wiphy->features |= NL80211_FEATURE_STATIC_SMPS; + wiphy->flags |= WIPHY_FLAG_IBSS_RSN; /* TODO: Check if HT capability advertised from firmware is different * for each band for a dual band capable radio. It will be tricky to * handle it when the ht capability different for each band. */ if (ht_cap & WMI_HT_CAP_DYNAMIC_SMPS) - ar->hw->wiphy->features |= NL80211_FEATURE_DYNAMIC_SMPS; + wiphy->features |= NL80211_FEATURE_DYNAMIC_SMPS; - ar->hw->wiphy->max_scan_ssids = WLAN_SCAN_PARAMS_MAX_SSID; - ar->hw->wiphy->max_scan_ie_len = WLAN_SCAN_PARAMS_MAX_IE_LEN; + wiphy->max_scan_ssids = WLAN_SCAN_PARAMS_MAX_SSID; + wiphy->max_scan_ie_len = WLAN_SCAN_PARAMS_MAX_IE_LEN; - ar->hw->max_listen_interval = ATH12K_MAX_HW_LISTEN_INTERVAL; + hw->max_listen_interval = ATH12K_MAX_HW_LISTEN_INTERVAL; - ar->hw->wiphy->flags |= WIPHY_FLAG_HAS_REMAIN_ON_CHANNEL; - ar->hw->wiphy->flags |= WIPHY_FLAG_HAS_CHANNEL_SWITCH; - ar->hw->wiphy->max_remain_on_channel_duration = 5000; + wiphy->flags |= WIPHY_FLAG_HAS_REMAIN_ON_CHANNEL; + wiphy->flags |= WIPHY_FLAG_HAS_CHANNEL_SWITCH; + wiphy->max_remain_on_channel_duration = 5000; - ar->hw->wiphy->flags |= WIPHY_FLAG_AP_UAPSD; - ar->hw->wiphy->features |= NL80211_FEATURE_AP_MODE_CHAN_WIDTH_CHANGE | + wiphy->flags |= WIPHY_FLAG_AP_UAPSD; + wiphy->features |= NL80211_FEATURE_AP_MODE_CHAN_WIDTH_CHANGE | NL80211_FEATURE_AP_SCAN; ar->max_num_stations = TARGET_NUM_STATIONS; ar->max_num_peers = TARGET_NUM_PEERS_PDEV; - ar->hw->wiphy->max_ap_assoc_sta = ar->max_num_stations; + wiphy->max_ap_assoc_sta = ar->max_num_stations; - ar->hw->queues = ATH12K_HW_MAX_QUEUES; - ar->hw->wiphy->tx_queue_len = ATH12K_QUEUE_LEN; - ar->hw->offchannel_tx_hw_queue = ATH12K_HW_MAX_QUEUES - 1; - ar->hw->max_rx_aggregation_subframes = IEEE80211_MAX_AMPDU_BUF_HE; + hw->queues = ATH12K_HW_MAX_QUEUES; + wiphy->tx_queue_len = ATH12K_QUEUE_LEN; + hw->offchannel_tx_hw_queue = ATH12K_HW_MAX_QUEUES - 1; + hw->max_rx_aggregation_subframes = IEEE80211_MAX_AMPDU_BUF_HE; - ar->hw->vif_data_size = sizeof(struct ath12k_vif); - ar->hw->sta_data_size = sizeof(struct ath12k_sta); + hw->vif_data_size = sizeof(struct ath12k_vif); + hw->sta_data_size = sizeof(struct ath12k_sta); - wiphy_ext_feature_set(ar->hw->wiphy, NL80211_EXT_FEATURE_CQM_RSSI_LIST); - wiphy_ext_feature_set(ar->hw->wiphy, NL80211_EXT_FEATURE_STA_TX_PWR); + wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_CQM_RSSI_LIST); + wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_STA_TX_PWR); - ar->hw->wiphy->cipher_suites = cipher_suites; - ar->hw->wiphy->n_cipher_suites = ARRAY_SIZE(cipher_suites); + wiphy->cipher_suites = cipher_suites; + wiphy->n_cipher_suites = ARRAY_SIZE(cipher_suites); - ar->hw->wiphy->iftype_ext_capab = ath12k_iftypes_ext_capa; - ar->hw->wiphy->num_iftype_ext_capab = - ARRAY_SIZE(ath12k_iftypes_ext_capa); + wiphy->iftype_ext_capab = ath12k_iftypes_ext_capa; + wiphy->num_iftype_ext_capab = ARRAY_SIZE(ath12k_iftypes_ext_capa); if (ar->supports_6ghz) { - wiphy_ext_feature_set(ar->hw->wiphy, + wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_FILS_DISCOVERY); - wiphy_ext_feature_set(ar->hw->wiphy, + wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_UNSOL_BCAST_PROBE_RESP); } - wiphy_ext_feature_set(ar->hw->wiphy, NL80211_EXT_FEATURE_PUNCT); + wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_PUNCT); - ath12k_reg_init(ar); + ath12k_reg_init(hw); if (!test_bit(ATH12K_FLAG_RAW_MODE, &ab->dev_flags)) { - ar->hw->netdev_features = NETIF_F_HW_CSUM; - ieee80211_hw_set(ar->hw, SW_CRYPTO_CONTROL); - ieee80211_hw_set(ar->hw, SUPPORT_FAST_XMIT); + hw->netdev_features = NETIF_F_HW_CSUM; + ieee80211_hw_set(hw, SW_CRYPTO_CONTROL); + ieee80211_hw_set(hw, SUPPORT_FAST_XMIT); } - ret = ieee80211_register_hw(ar->hw); + ret = ieee80211_register_hw(hw); if (ret) { ath12k_err(ar->ab, "ieee80211 registration failed: %d\n", ret); goto err_free_if_combs; @@ -7518,7 +7534,7 @@ static int __ath12k_mac_register(struct ath12k *ar) * while. But that time is so short and in practise it make * a difference in real life. */ - ar->hw->wiphy->interface_modes &= ~BIT(NL80211_IFTYPE_MONITOR); + wiphy->interface_modes &= ~BIT(NL80211_IFTYPE_MONITOR); /* Apply the regd received during initialization */ ret = ath12k_regd_update(ar, true); @@ -7530,11 +7546,11 @@ static int __ath12k_mac_register(struct ath12k *ar) return 0; err_unregister_hw: - ieee80211_unregister_hw(ar->hw); + ieee80211_unregister_hw(hw); err_free_if_combs: - kfree(ar->hw->wiphy->iface_combinations[0].limits); - kfree(ar->hw->wiphy->iface_combinations); + kfree(wiphy->iface_combinations[0].limits); + kfree(wiphy->iface_combinations); err_free_channels: kfree(ar->mac.sbands[NL80211_BAND_2GHZ].channels); @@ -7542,7 +7558,7 @@ static int __ath12k_mac_register(struct ath12k *ar) kfree(ar->mac.sbands[NL80211_BAND_6GHZ].channels); err: - SET_IEEE80211_DEV(ar->hw, NULL); + SET_IEEE80211_DEV(hw, NULL); return ret; } diff --git a/drivers/net/wireless/ath/ath12k/mac.h b/drivers/net/wireless/ath/ath12k/mac.h index 59b4e8f5eee059..7c63bb628adc1b 100644 --- a/drivers/net/wireless/ath/ath12k/mac.h +++ b/drivers/net/wireless/ath/ath12k/mac.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef ATH12K_MAC_H @@ -43,6 +43,7 @@ enum ath12k_supported_bw { ATH12K_BW_40 = 1, ATH12K_BW_80 = 2, ATH12K_BW_160 = 3, + ATH12K_BW_320 = 4, }; extern const struct htt_rx_ring_tlv_filter ath12k_mac_mon_status_filter_default; diff --git a/drivers/net/wireless/ath/ath12k/mhi.c b/drivers/net/wireless/ath/ath12k/mhi.c index 39e640293cdc0d..d5441ddb374b08 100644 --- a/drivers/net/wireless/ath/ath12k/mhi.c +++ b/drivers/net/wireless/ath/ath12k/mhi.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2020-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include @@ -251,6 +251,7 @@ static int ath12k_mhi_get_msi(struct ath12k_pci *ab_pci) u32 user_base_data, base_vector; int ret, num_vectors, i; int *irq; + unsigned int msi_data; ret = ath12k_pci_get_user_msi_assignment(ab, "MHI", &num_vectors, @@ -265,9 +266,15 @@ static int ath12k_mhi_get_msi(struct ath12k_pci *ab_pci) if (!irq) return -ENOMEM; - for (i = 0; i < num_vectors; i++) - irq[i] = ath12k_pci_get_msi_irq(ab->dev, - base_vector + i); + msi_data = base_vector; + for (i = 0; i < num_vectors; i++) { + if (test_bit(ATH12K_PCI_FLAG_MULTI_MSI_VECTORS, &ab_pci->flags)) + irq[i] = ath12k_pci_get_msi_irq(ab->dev, + msi_data++); + else + irq[i] = ath12k_pci_get_msi_irq(ab->dev, + msi_data); + } ab_pci->mhi_ctrl->irq = irq; ab_pci->mhi_ctrl->nr_irqs = num_vectors; @@ -374,6 +381,9 @@ int ath12k_mhi_register(struct ath12k_pci *ab_pci) goto free_controller; } + if (!test_bit(ATH12K_PCI_FLAG_MULTI_MSI_VECTORS, &ab_pci->flags)) + mhi_ctrl->irq_flags = IRQF_SHARED | IRQF_NOBALANCING; + mhi_ctrl->iova_start = 0; mhi_ctrl->iova_stop = 0xffffffff; mhi_ctrl->sbl_size = SZ_512K; diff --git a/drivers/net/wireless/ath/ath12k/pci.c b/drivers/net/wireless/ath/ath12k/pci.c index 3006cd3fbe1192..f0d2e2d8719c00 100644 --- a/drivers/net/wireless/ath/ath12k/pci.c +++ b/drivers/net/wireless/ath/ath12k/pci.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2019-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include @@ -60,6 +60,17 @@ static const struct ath12k_msi_config ath12k_msi_config[] = { }, }; +static const struct ath12k_msi_config msi_config_one_msi = { + .total_vectors = 1, + .total_users = 4, + .users = (struct ath12k_msi_user[]) { + { .name = "MHI", .num_vectors = 3, .base_vector = 0 }, + { .name = "CE", .num_vectors = 1, .base_vector = 0 }, + { .name = "WAKE", .num_vectors = 1, .base_vector = 0 }, + { .name = "DP", .num_vectors = 1, .base_vector = 0 }, + }, +}; + static const char *irq_name[ATH12K_IRQ_NUM_MAX] = { "bhi", "mhi-er0", @@ -355,16 +366,30 @@ static void ath12k_pci_free_irq(struct ath12k_base *ab) static void ath12k_pci_ce_irq_enable(struct ath12k_base *ab, u16 ce_id) { + struct ath12k_pci *ab_pci = ath12k_pci_priv(ab); u32 irq_idx; + /* In case of one MSI vector, we handle irq enable/disable in a + * uniform way since we only have one irq + */ + if (!test_bit(ATH12K_PCI_FLAG_MULTI_MSI_VECTORS, &ab_pci->flags)) + return; + irq_idx = ATH12K_PCI_IRQ_CE0_OFFSET + ce_id; enable_irq(ab->irq_num[irq_idx]); } static void ath12k_pci_ce_irq_disable(struct ath12k_base *ab, u16 ce_id) { + struct ath12k_pci *ab_pci = ath12k_pci_priv(ab); u32 irq_idx; + /* In case of one MSI vector, we handle irq enable/disable in a + * uniform way since we only have one irq + */ + if (!test_bit(ATH12K_PCI_FLAG_MULTI_MSI_VECTORS, &ab_pci->flags)) + return; + irq_idx = ATH12K_PCI_IRQ_CE0_OFFSET + ce_id; disable_irq_nosync(ab->irq_num[irq_idx]); } @@ -373,6 +398,8 @@ static void ath12k_pci_ce_irqs_disable(struct ath12k_base *ab) { int i; + clear_bit(ATH12K_FLAG_CE_IRQ_ENABLED, &ab->dev_flags); + for (i = 0; i < ab->hw_params->ce_count; i++) { if (ath12k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR) continue; @@ -397,20 +424,27 @@ static void ath12k_pci_sync_ce_irqs(struct ath12k_base *ab) static void ath12k_pci_ce_tasklet(struct tasklet_struct *t) { struct ath12k_ce_pipe *ce_pipe = from_tasklet(ce_pipe, t, intr_tq); + int irq_idx = ATH12K_PCI_IRQ_CE0_OFFSET + ce_pipe->pipe_num; ath12k_ce_per_engine_service(ce_pipe->ab, ce_pipe->pipe_num); - ath12k_pci_ce_irq_enable(ce_pipe->ab, ce_pipe->pipe_num); + enable_irq(ce_pipe->ab->irq_num[irq_idx]); } static irqreturn_t ath12k_pci_ce_interrupt_handler(int irq, void *arg) { struct ath12k_ce_pipe *ce_pipe = arg; + struct ath12k_base *ab = ce_pipe->ab; + int irq_idx = ATH12K_PCI_IRQ_CE0_OFFSET + ce_pipe->pipe_num; + + if (!test_bit(ATH12K_FLAG_CE_IRQ_ENABLED, &ab->dev_flags)) + return IRQ_HANDLED; /* last interrupt received for this CE */ ce_pipe->timestamp = jiffies; - ath12k_pci_ce_irq_disable(ce_pipe->ab, ce_pipe->pipe_num); + disable_irq_nosync(ab->irq_num[irq_idx]); + tasklet_schedule(&ce_pipe->intr_tq); return IRQ_HANDLED; @@ -418,8 +452,15 @@ static irqreturn_t ath12k_pci_ce_interrupt_handler(int irq, void *arg) static void ath12k_pci_ext_grp_disable(struct ath12k_ext_irq_grp *irq_grp) { + struct ath12k_pci *ab_pci = ath12k_pci_priv(irq_grp->ab); int i; + /* In case of one MSI vector, we handle irq enable/disable + * in a uniform way since we only have one irq + */ + if (!test_bit(ATH12K_PCI_FLAG_MULTI_MSI_VECTORS, &ab_pci->flags)) + return; + for (i = 0; i < irq_grp->num_irq; i++) disable_irq_nosync(irq_grp->ab->irq_num[irq_grp->irqs[i]]); } @@ -428,6 +469,8 @@ static void __ath12k_pci_ext_irq_disable(struct ath12k_base *ab) { int i; + clear_bit(ATH12K_FLAG_EXT_IRQ_ENABLED, &ab->dev_flags); + for (i = 0; i < ATH12K_EXT_IRQ_GRP_NUM_MAX; i++) { struct ath12k_ext_irq_grp *irq_grp = &ab->ext_irq_grp[i]; @@ -440,8 +483,15 @@ static void __ath12k_pci_ext_irq_disable(struct ath12k_base *ab) static void ath12k_pci_ext_grp_enable(struct ath12k_ext_irq_grp *irq_grp) { + struct ath12k_pci *ab_pci = ath12k_pci_priv(irq_grp->ab); int i; + /* In case of one MSI vector, we handle irq enable/disable in a + * uniform way since we only have one irq + */ + if (!test_bit(ATH12K_PCI_FLAG_MULTI_MSI_VECTORS, &ab_pci->flags)) + return; + for (i = 0; i < irq_grp->num_irq; i++) enable_irq(irq_grp->ab->irq_num[irq_grp->irqs[i]]); } @@ -467,11 +517,13 @@ static int ath12k_pci_ext_grp_napi_poll(struct napi_struct *napi, int budget) napi); struct ath12k_base *ab = irq_grp->ab; int work_done; + int i; work_done = ath12k_dp_service_srng(ab, irq_grp, budget); if (work_done < budget) { napi_complete_done(napi, work_done); - ath12k_pci_ext_grp_enable(irq_grp); + for (i = 0; i < irq_grp->num_irq; i++) + enable_irq(irq_grp->ab->irq_num[irq_grp->irqs[i]]); } if (work_done > budget) @@ -483,13 +535,19 @@ static int ath12k_pci_ext_grp_napi_poll(struct napi_struct *napi, int budget) static irqreturn_t ath12k_pci_ext_interrupt_handler(int irq, void *arg) { struct ath12k_ext_irq_grp *irq_grp = arg; + struct ath12k_base *ab = irq_grp->ab; + int i; + + if (!test_bit(ATH12K_FLAG_EXT_IRQ_ENABLED, &ab->dev_flags)) + return IRQ_HANDLED; ath12k_dbg(irq_grp->ab, ATH12K_DBG_PCI, "ext irq:%d\n", irq); /* last interrupt received for this group */ irq_grp->timestamp = jiffies; - ath12k_pci_ext_grp_disable(irq_grp); + for (i = 0; i < irq_grp->num_irq; i++) + disable_irq_nosync(irq_grp->ab->irq_num[irq_grp->irqs[i]]); napi_schedule(&irq_grp->napi); @@ -498,6 +556,7 @@ static irqreturn_t ath12k_pci_ext_interrupt_handler(int irq, void *arg) static int ath12k_pci_ext_irq_config(struct ath12k_base *ab) { + struct ath12k_pci *ab_pci = ath12k_pci_priv(ab); int i, j, ret, num_vectors = 0; u32 user_base_data = 0, base_vector = 0, base_idx; @@ -544,23 +603,32 @@ static int ath12k_pci_ext_irq_config(struct ath12k_base *ab) irq_set_status_flags(irq, IRQ_DISABLE_UNLAZY); ret = request_irq(irq, ath12k_pci_ext_interrupt_handler, - IRQF_SHARED, + ab_pci->irq_flags, "DP_EXT_IRQ", irq_grp); if (ret) { ath12k_err(ab, "failed request irq %d: %d\n", vector, ret); return ret; } - - disable_irq_nosync(ab->irq_num[irq_idx]); } + ath12k_pci_ext_grp_disable(irq_grp); } return 0; } +static int ath12k_pci_set_irq_affinity_hint(struct ath12k_pci *ab_pci, + const struct cpumask *m) +{ + if (test_bit(ATH12K_PCI_FLAG_MULTI_MSI_VECTORS, &ab_pci->flags)) + return 0; + + return irq_set_affinity_hint(ab_pci->pdev->irq, m); +} + static int ath12k_pci_config_irq(struct ath12k_base *ab) { + struct ath12k_pci *ab_pci = ath12k_pci_priv(ab); struct ath12k_ce_pipe *ce_pipe; u32 msi_data_start; u32 msi_data_count, msi_data_idx; @@ -589,7 +657,7 @@ static int ath12k_pci_config_irq(struct ath12k_base *ab) tasklet_setup(&ce_pipe->intr_tq, ath12k_pci_ce_tasklet); ret = request_irq(irq, ath12k_pci_ce_interrupt_handler, - IRQF_SHARED, irq_name[irq_idx], + ab_pci->irq_flags, irq_name[irq_idx], ce_pipe); if (ret) { ath12k_err(ab, "failed to request irq %d: %d\n", @@ -626,6 +694,8 @@ static void ath12k_pci_ce_irqs_enable(struct ath12k_base *ab) { int i; + set_bit(ATH12K_FLAG_CE_IRQ_ENABLED, &ab->dev_flags); + for (i = 0; i < ab->hw_params->ce_count; i++) { if (ath12k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR) continue; @@ -670,16 +740,27 @@ static int ath12k_pci_msi_alloc(struct ath12k_pci *ab_pci) msi_config->total_vectors, msi_config->total_vectors, PCI_IRQ_MSI); - if (num_vectors != msi_config->total_vectors) { - ath12k_err(ab, "failed to get %d MSI vectors, only %d available", - msi_config->total_vectors, num_vectors); - if (num_vectors >= 0) - return -EINVAL; - else - return num_vectors; + if (num_vectors == msi_config->total_vectors) { + set_bit(ATH12K_PCI_FLAG_MULTI_MSI_VECTORS, &ab_pci->flags); + ab_pci->irq_flags = IRQF_SHARED; + } else { + num_vectors = pci_alloc_irq_vectors(ab_pci->pdev, + 1, + 1, + PCI_IRQ_MSI); + if (num_vectors < 0) { + ret = -EINVAL; + goto reset_msi_config; + } + clear_bit(ATH12K_PCI_FLAG_MULTI_MSI_VECTORS, &ab_pci->flags); + ab_pci->msi_config = &msi_config_one_msi; + ab_pci->irq_flags = IRQF_SHARED | IRQF_NOBALANCING; + ath12k_dbg(ab, ATH12K_DBG_PCI, "request MSI one vector\n"); } + ath12k_info(ab, "MSI vectors: %d\n", num_vectors); + ath12k_pci_msi_disable(ab_pci); msi_desc = irq_get_msi_desc(ab_pci->pdev->irq); @@ -700,6 +781,7 @@ static int ath12k_pci_msi_alloc(struct ath12k_pci *ab_pci) free_msi_vector: pci_free_irq_vectors(ab_pci->pdev); +reset_msi_config: return ret; } @@ -708,6 +790,25 @@ static void ath12k_pci_msi_free(struct ath12k_pci *ab_pci) pci_free_irq_vectors(ab_pci->pdev); } +static int ath12k_pci_config_msi_data(struct ath12k_pci *ab_pci) +{ + struct msi_desc *msi_desc; + + msi_desc = irq_get_msi_desc(ab_pci->pdev->irq); + if (!msi_desc) { + ath12k_err(ab_pci->ab, "msi_desc is NULL!\n"); + pci_free_irq_vectors(ab_pci->pdev); + return -EINVAL; + } + + ab_pci->msi_ep_base_data = msi_desc->msg.data; + + ath12k_dbg(ab_pci->ab, ATH12K_DBG_PCI, "pci after request_irq msi_ep_base_data %d\n", + ab_pci->msi_ep_base_data); + + return 0; +} + static int ath12k_pci_claim(struct ath12k_pci *ab_pci, struct pci_dev *pdev) { struct ath12k_base *ab = ab_pci->ab; @@ -891,11 +992,11 @@ int ath12k_pci_get_user_msi_assignment(struct ath12k_base *ab, char *user_name, for (idx = 0; idx < msi_config->total_users; idx++) { if (strcmp(user_name, msi_config->users[idx].name) == 0) { *num_vectors = msi_config->users[idx].num_vectors; - *user_base_data = msi_config->users[idx].base_vector - + ab_pci->msi_ep_base_data; - *base_vector = msi_config->users[idx].base_vector; + *base_vector = msi_config->users[idx].base_vector; + *user_base_data = *base_vector + ab_pci->msi_ep_base_data; - ath12k_dbg(ab, ATH12K_DBG_PCI, "Assign MSI to user: %s, num_vectors: %d, user_base_data: %u, base_vector: %u\n", + ath12k_dbg(ab, ATH12K_DBG_PCI, + "Assign MSI to user: %s, num_vectors: %d, user_base_data: %u, base_vector: %u\n", user_name, *num_vectors, *user_base_data, *base_vector); @@ -956,6 +1057,8 @@ void ath12k_pci_ext_irq_enable(struct ath12k_base *ab) { int i; + set_bit(ATH12K_FLAG_EXT_IRQ_ENABLED, &ab->dev_flags); + for (i = 0; i < ATH12K_EXT_IRQ_GRP_NUM_MAX; i++) { struct ath12k_ext_irq_grp *irq_grp = &ab->ext_irq_grp[i]; @@ -1000,7 +1103,10 @@ int ath12k_pci_start(struct ath12k_base *ab) set_bit(ATH12K_PCI_FLAG_INIT_DONE, &ab_pci->flags); - ath12k_pci_aspm_restore(ab_pci); + if (test_bit(ATH12K_PCI_FLAG_MULTI_MSI_VECTORS, &ab_pci->flags)) + ath12k_pci_aspm_restore(ab_pci); + else + ath12k_info(ab, "leaving PCI ASPM disabled to avoid MHI M2 problems\n"); ath12k_pci_ce_irqs_enable(ab); ath12k_ce_rx_post_buf(ab); @@ -1262,10 +1368,16 @@ static int ath12k_pci_probe(struct pci_dev *pdev, if (ret) goto err_pci_msi_free; + ret = ath12k_pci_set_irq_affinity_hint(ab_pci, cpumask_of(0)); + if (ret) { + ath12k_err(ab, "failed to set irq affinity %d\n", ret); + goto err_pci_msi_free; + } + ret = ath12k_mhi_register(ab_pci); if (ret) { ath12k_err(ab, "failed to register mhi: %d\n", ret); - goto err_pci_msi_free; + goto err_irq_affinity_cleanup; } ret = ath12k_hal_srng_init(ab); @@ -1286,6 +1398,17 @@ static int ath12k_pci_probe(struct pci_dev *pdev, goto err_ce_free; } + /* kernel may allocate a dummy vector before request_irq and + * then allocate a real vector when request_irq is called. + * So get msi_data here again to avoid spurious interrupt + * as msi_data will configured to srngs. + */ + ret = ath12k_pci_config_msi_data(ab_pci); + if (ret) { + ath12k_err(ab, "failed to config msi_data: %d\n", ret); + goto err_free_irq; + } + ret = ath12k_core_init(ab); if (ret) { ath12k_err(ab, "failed to init core: %d\n", ret); @@ -1308,6 +1431,9 @@ static int ath12k_pci_probe(struct pci_dev *pdev, err_pci_msi_free: ath12k_pci_msi_free(ab_pci); +err_irq_affinity_cleanup: + ath12k_pci_set_irq_affinity_hint(ab_pci, NULL); + err_pci_free_region: ath12k_pci_free_region(ab_pci); @@ -1322,6 +1448,8 @@ static void ath12k_pci_remove(struct pci_dev *pdev) struct ath12k_base *ab = pci_get_drvdata(pdev); struct ath12k_pci *ab_pci = ath12k_pci_priv(ab); + ath12k_pci_set_irq_affinity_hint(ab_pci, NULL); + if (test_bit(ATH12K_FLAG_QMI_FAIL, &ab->dev_flags)) { ath12k_pci_power_down(ab); ath12k_qmi_deinit_service(ab); @@ -1348,7 +1476,9 @@ static void ath12k_pci_remove(struct pci_dev *pdev) static void ath12k_pci_shutdown(struct pci_dev *pdev) { struct ath12k_base *ab = pci_get_drvdata(pdev); + struct ath12k_pci *ab_pci = ath12k_pci_priv(ab); + ath12k_pci_set_irq_affinity_hint(ab_pci, NULL); ath12k_pci_power_down(ab); } diff --git a/drivers/net/wireless/ath/ath12k/pci.h b/drivers/net/wireless/ath/ath12k/pci.h index 0f24fd9395cd9b..b2edf32ada20a9 100644 --- a/drivers/net/wireless/ath/ath12k/pci.h +++ b/drivers/net/wireless/ath/ath12k/pci.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2019-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef ATH12K_PCI_H #define ATH12K_PCI_H @@ -84,6 +84,7 @@ enum ath12k_pci_flags { ATH12K_PCI_FLAG_INIT_DONE, ATH12K_PCI_FLAG_IS_MSI_64, ATH12K_PCI_ASPM_RESTORE, + ATH12K_PCI_FLAG_MULTI_MSI_VECTORS, }; struct ath12k_pci_ops { @@ -108,6 +109,7 @@ struct ath12k_pci { /* enum ath12k_pci_flags */ unsigned long flags; u16 link_ctl; + unsigned long irq_flags; const struct ath12k_pci_ops *pci_ops; }; diff --git a/drivers/net/wireless/ath/ath12k/peer.h b/drivers/net/wireless/ath/ath12k/peer.h index c6edb24cbedd88..7b3500b5c8c20e 100644 --- a/drivers/net/wireless/ath/ath12k/peer.h +++ b/drivers/net/wireless/ath/ath12k/peer.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef ATH12K_PEER_H diff --git a/drivers/net/wireless/ath/ath12k/qmi.c b/drivers/net/wireless/ath/ath12k/qmi.c index f6e949c618d0ae..77a132f6bbd1be 100644 --- a/drivers/net/wireless/ath/ath12k/qmi.c +++ b/drivers/net/wireless/ath/ath12k/qmi.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include diff --git a/drivers/net/wireless/ath/ath12k/qmi.h b/drivers/net/wireless/ath/ath12k/qmi.h index e20d6511d1ca04..e25bbaa125e833 100644 --- a/drivers/net/wireless/ath/ath12k/qmi.h +++ b/drivers/net/wireless/ath/ath12k/qmi.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef ATH12K_QMI_H diff --git a/drivers/net/wireless/ath/ath12k/reg.c b/drivers/net/wireless/ath/ath12k/reg.c index 5c006256c82ad1..f924bc13ccff5f 100644 --- a/drivers/net/wireless/ath/ath12k/reg.c +++ b/drivers/net/wireless/ath/ath12k/reg.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include #include "core.h" @@ -28,11 +28,11 @@ static const struct ieee80211_regdomain ath12k_world_regd = { } }; -static bool ath12k_regdom_changes(struct ath12k *ar, char *alpha2) +static bool ath12k_regdom_changes(struct ieee80211_hw *hw, char *alpha2) { const struct ieee80211_regdomain *regd; - regd = rcu_dereference_rtnl(ar->hw->wiphy->regd); + regd = rcu_dereference_rtnl(hw->wiphy->regd); /* This can happen during wiphy registration where the previous * user request is received before we update the regd received * from firmware. @@ -71,7 +71,7 @@ ath12k_reg_notifier(struct wiphy *wiphy, struct regulatory_request *request) return; } - if (!ath12k_regdom_changes(ar, request->alpha2)) { + if (!ath12k_regdom_changes(hw, request->alpha2)) { ath12k_dbg(ar->ab, ATH12K_DBG_REG, "Country is already set\n"); return; } @@ -199,6 +199,7 @@ static void ath12k_copy_regd(struct ieee80211_regdomain *regd_orig, int ath12k_regd_update(struct ath12k *ar, bool init) { + struct ieee80211_hw *hw = ar->hw; struct ieee80211_regdomain *regd, *regd_copy = NULL; int ret, regd_len, pdev_id; struct ath12k_base *ab; @@ -246,9 +247,9 @@ int ath12k_regd_update(struct ath12k *ar, bool init) } rtnl_lock(); - wiphy_lock(ar->hw->wiphy); - ret = regulatory_set_wiphy_regd_sync(ar->hw->wiphy, regd_copy); - wiphy_unlock(ar->hw->wiphy); + wiphy_lock(hw->wiphy); + ret = regulatory_set_wiphy_regd_sync(hw->wiphy, regd_copy); + wiphy_unlock(hw->wiphy); rtnl_unlock(); kfree(regd_copy); @@ -729,10 +730,10 @@ void ath12k_regd_update_work(struct work_struct *work) } } -void ath12k_reg_init(struct ath12k *ar) +void ath12k_reg_init(struct ieee80211_hw *hw) { - ar->hw->wiphy->regulatory_flags = REGULATORY_WIPHY_SELF_MANAGED; - ar->hw->wiphy->reg_notifier = ath12k_reg_notifier; + hw->wiphy->regulatory_flags = REGULATORY_WIPHY_SELF_MANAGED; + hw->wiphy->reg_notifier = ath12k_reg_notifier; } void ath12k_reg_free(struct ath12k_base *ab) diff --git a/drivers/net/wireless/ath/ath12k/reg.h b/drivers/net/wireless/ath/ath12k/reg.h index 35569f03042d3d..29c7ec3260dafe 100644 --- a/drivers/net/wireless/ath/ath12k/reg.h +++ b/drivers/net/wireless/ath/ath12k/reg.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2019-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef ATH12K_REG_H @@ -89,7 +89,7 @@ enum ath12k_reg_phy_bitmap { ATH12K_REG_PHY_BITMAP_NO11BE = BIT(6), }; -void ath12k_reg_init(struct ath12k *ar); +void ath12k_reg_init(struct ieee80211_hw *hw); void ath12k_reg_free(struct ath12k_base *ab); void ath12k_regd_update_work(struct work_struct *work); struct ieee80211_regdomain *ath12k_reg_build_regd(struct ath12k_base *ab, diff --git a/drivers/net/wireless/ath/ath12k/rx_desc.h b/drivers/net/wireless/ath/ath12k/rx_desc.h index c4058abc516ee0..55f20c446ca9c8 100644 --- a/drivers/net/wireless/ath/ath12k/rx_desc.h +++ b/drivers/net/wireless/ath/ath12k/rx_desc.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef ATH12K_RX_DESC_H #define ATH12K_RX_DESC_H diff --git a/drivers/net/wireless/ath/ath12k/wmi.c b/drivers/net/wireless/ath/ath12k/wmi.c index 0e5bf5ce8d4c38..11cc3005c0f983 100644 --- a/drivers/net/wireless/ath/ath12k/wmi.c +++ b/drivers/net/wireless/ath/ath12k/wmi.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #include #include diff --git a/drivers/net/wireless/ath/ath12k/wmi.h b/drivers/net/wireless/ath/ath12k/wmi.h index 629373d6742120..06e5b9b4049b0b 100644 --- a/drivers/net/wireless/ath/ath12k/wmi.h +++ b/drivers/net/wireless/ath/ath12k/wmi.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef ATH12K_WMI_H @@ -1146,25 +1146,27 @@ enum wmi_tlv_vdev_param { }; enum wmi_tlv_peer_flags { - WMI_TLV_PEER_AUTH = 0x00000001, - WMI_TLV_PEER_QOS = 0x00000002, - WMI_TLV_PEER_NEED_PTK_4_WAY = 0x00000004, - WMI_TLV_PEER_NEED_GTK_2_WAY = 0x00000010, - WMI_TLV_PEER_APSD = 0x00000800, - WMI_TLV_PEER_HT = 0x00001000, - WMI_TLV_PEER_40MHZ = 0x00002000, - WMI_TLV_PEER_STBC = 0x00008000, - WMI_TLV_PEER_LDPC = 0x00010000, - WMI_TLV_PEER_DYN_MIMOPS = 0x00020000, - WMI_TLV_PEER_STATIC_MIMOPS = 0x00040000, - WMI_TLV_PEER_SPATIAL_MUX = 0x00200000, - WMI_TLV_PEER_VHT = 0x02000000, - WMI_TLV_PEER_80MHZ = 0x04000000, - WMI_TLV_PEER_PMF = 0x08000000, + WMI_PEER_AUTH = 0x00000001, + WMI_PEER_QOS = 0x00000002, + WMI_PEER_NEED_PTK_4_WAY = 0x00000004, + WMI_PEER_NEED_GTK_2_WAY = 0x00000010, + WMI_PEER_HE = 0x00000400, + WMI_PEER_APSD = 0x00000800, + WMI_PEER_HT = 0x00001000, + WMI_PEER_40MHZ = 0x00002000, + WMI_PEER_STBC = 0x00008000, + WMI_PEER_LDPC = 0x00010000, + WMI_PEER_DYN_MIMOPS = 0x00020000, + WMI_PEER_STATIC_MIMOPS = 0x00040000, + WMI_PEER_SPATIAL_MUX = 0x00200000, + WMI_PEER_TWT_REQ = 0x00400000, + WMI_PEER_TWT_RESP = 0x00800000, + WMI_PEER_VHT = 0x02000000, + WMI_PEER_80MHZ = 0x04000000, + WMI_PEER_PMF = 0x08000000, WMI_PEER_IS_P2P_CAPABLE = 0x20000000, WMI_PEER_160MHZ = 0x40000000, WMI_PEER_SAFEMODE_EN = 0x80000000, - }; enum wmi_tlv_peer_flags_ext { @@ -2220,6 +2222,7 @@ enum wmi_peer_chwidth { WMI_PEER_CHWIDTH_40MHZ = 1, WMI_PEER_CHWIDTH_80MHZ = 2, WMI_PEER_CHWIDTH_160MHZ = 3, + WMI_PEER_CHWIDTH_320MHZ = 4, }; enum wmi_beacon_gen_mode { @@ -3844,31 +3847,6 @@ struct wmi_unit_test_cmd { #define MAX_SUPPORTED_RATES 128 -#define WMI_PEER_AUTH 0x00000001 -#define WMI_PEER_QOS 0x00000002 -#define WMI_PEER_NEED_PTK_4_WAY 0x00000004 -#define WMI_PEER_NEED_GTK_2_WAY 0x00000010 -#define WMI_PEER_HE 0x00000400 -#define WMI_PEER_APSD 0x00000800 -#define WMI_PEER_HT 0x00001000 -#define WMI_PEER_40MHZ 0x00002000 -#define WMI_PEER_STBC 0x00008000 -#define WMI_PEER_LDPC 0x00010000 -#define WMI_PEER_DYN_MIMOPS 0x00020000 -#define WMI_PEER_STATIC_MIMOPS 0x00040000 -#define WMI_PEER_SPATIAL_MUX 0x00200000 -#define WMI_PEER_TWT_REQ 0x00400000 -#define WMI_PEER_TWT_RESP 0x00800000 -#define WMI_PEER_VHT 0x02000000 -#define WMI_PEER_80MHZ 0x04000000 -#define WMI_PEER_PMF 0x08000000 -/* TODO: Place holder for WLAN_PEER_F_PS_PRESEND_REQUIRED = 0x10000000. - * Need to be cleaned up - */ -#define WMI_PEER_IS_P2P_CAPABLE 0x20000000 -#define WMI_PEER_160MHZ 0x40000000 -#define WMI_PEER_SAFEMODE_EN 0x80000000 - struct ath12k_wmi_vht_rate_set_params { __le32 tlv_header; __le32 rx_max_rate; @@ -4770,7 +4748,6 @@ struct wmi_probe_tmpl_cmd { struct ath12k_wmi_pdev { struct ath12k_wmi_base *wmi_ab; enum ath12k_htc_ep_id eid; - const struct wmi_peer_flags_map *peer_flags; u32 rx_decap_mode; }; @@ -4784,7 +4761,6 @@ struct ath12k_wmi_base { struct completion unified_ready; DECLARE_BITMAP(svc_map, WMI_MAX_EXT2_SERVICE); wait_queue_head_t tx_credits_wq; - const struct wmi_peer_flags_map *peer_flags; u32 num_mem_chunks; u32 rx_decap_mode; struct ath12k_wmi_host_mem_chunk_arg mem_chunks[WMI_MAX_MEM_REQS]; diff --git a/drivers/net/wireless/ath/ath5k/ahb.c b/drivers/net/wireless/ath/ath5k/ahb.c index 08bd5d3b00f104..f27308ccb2f15e 100644 --- a/drivers/net/wireless/ath/ath5k/ahb.c +++ b/drivers/net/wireless/ath/ath5k/ahb.c @@ -185,7 +185,7 @@ static int ath_ahb_probe(struct platform_device *pdev) return ret; } -static int ath_ahb_remove(struct platform_device *pdev) +static void ath_ahb_remove(struct platform_device *pdev) { struct ar231x_board_config *bcfg = dev_get_platdata(&pdev->dev); struct ieee80211_hw *hw = platform_get_drvdata(pdev); @@ -193,7 +193,7 @@ static int ath_ahb_remove(struct platform_device *pdev) u32 reg; if (!hw) - return 0; + return; ah = hw->priv; @@ -215,13 +215,11 @@ static int ath_ahb_remove(struct platform_device *pdev) ath5k_deinit_ah(ah); iounmap(ah->iobase); ieee80211_free_hw(hw); - - return 0; } static struct platform_driver ath_ahb_driver = { .probe = ath_ahb_probe, - .remove = ath_ahb_remove, + .remove_new = ath_ahb_remove, .driver = { .name = "ar231x-wmac", }, diff --git a/drivers/net/wireless/ath/ath5k/eeprom.h b/drivers/net/wireless/ath/ath5k/eeprom.h index 693296ee9693f8..e85b713950b161 100644 --- a/drivers/net/wireless/ath/ath5k/eeprom.h +++ b/drivers/net/wireless/ath/ath5k/eeprom.h @@ -489,7 +489,4 @@ struct ath5k_eeprom_info { /* Spur mitigation data (fbin values for spur channels) */ u16 ee_spur_chans[AR5K_EEPROM_N_SPUR_CHANS][AR5K_EEPROM_N_FREQ_BANDS]; - - /* Antenna raw switch tables */ - u32 ee_antenna[AR5K_EEPROM_N_MODES][AR5K_ANT_MAX]; }; diff --git a/drivers/net/wireless/ath/ath9k/ath9k_pci_owl_loader.c b/drivers/net/wireless/ath/ath9k/ath9k_pci_owl_loader.c index 708c8969b503ab..a5eb43f3032013 100644 --- a/drivers/net/wireless/ath/ath9k/ath9k_pci_owl_loader.c +++ b/drivers/net/wireless/ath/ath9k/ath9k_pci_owl_loader.c @@ -125,7 +125,7 @@ static void owl_rescan(struct pci_dev *pdev) static void owl_fw_cb(const struct firmware *fw, void *context) { - struct owl_ctx *ctx = (struct owl_ctx *)context; + struct owl_ctx *ctx = context; complete(&ctx->eeprom_load); diff --git a/drivers/net/wireless/ath/ath9k/common-init.c b/drivers/net/wireless/ath/ath9k/common-init.c index 82de0fadbc956d..7c13a1deb3acf1 100644 --- a/drivers/net/wireless/ath/ath9k/common-init.c +++ b/drivers/net/wireless/ath/ath9k/common-init.c @@ -124,7 +124,7 @@ static struct ieee80211_rate ath9k_legacy_rates[] = { int ath9k_cmn_init_channels_rates(struct ath_common *common) { - struct ath_hw *ah = (struct ath_hw *)common->ah; + struct ath_hw *ah = common->ah; void *channels; BUILD_BUG_ON(ARRAY_SIZE(ath9k_2ghz_chantable) + diff --git a/drivers/net/wireless/ath/ath9k/common-spectral.c b/drivers/net/wireless/ath/ath9k/common-spectral.c index a5349c72c3321f..4b27445a5fb87d 100644 --- a/drivers/net/wireless/ath/ath9k/common-spectral.c +++ b/drivers/net/wireless/ath/ath9k/common-spectral.c @@ -471,7 +471,7 @@ int ath_cmn_process_fft(struct ath_spec_scan_priv *spec_priv, struct ieee80211_h u8 sample_buf[SPECTRAL_SAMPLE_MAX_LEN] = {0}; struct ath_hw *ah = spec_priv->ah; struct ath_common *common = ath9k_hw_common(spec_priv->ah); - struct ath_softc *sc = (struct ath_softc *)common->priv; + struct ath_softc *sc = common->priv; u8 num_bins, *vdata = (u8 *)hdr; struct ath_radar_info *radar_info; int len = rs->rs_datalen; diff --git a/drivers/net/wireless/ath/ath9k/debug.c b/drivers/net/wireless/ath/ath9k/debug.c index a0376a6787b8d0..d84e3ee7b5d902 100644 --- a/drivers/net/wireless/ath/ath9k/debug.c +++ b/drivers/net/wireless/ath/ath9k/debug.c @@ -1376,7 +1376,7 @@ void ath9k_deinit_debug(struct ath_softc *sc) int ath9k_init_debug(struct ath_hw *ah) { struct ath_common *common = ath9k_hw_common(ah); - struct ath_softc *sc = (struct ath_softc *) common->priv; + struct ath_softc *sc = common->priv; sc->debug.debugfs_phy = debugfs_create_dir("ath9k", sc->hw->wiphy->debugfsdir); diff --git a/drivers/net/wireless/ath/ath9k/hif_usb.c b/drivers/net/wireless/ath/ath9k/hif_usb.c index 90cfe39aa433c9..0c7841f952287f 100644 --- a/drivers/net/wireless/ath/ath9k/hif_usb.c +++ b/drivers/net/wireless/ath/ath9k/hif_usb.c @@ -70,7 +70,7 @@ static int __hif_usb_tx(struct hif_device_usb *hif_dev); static void hif_usb_regout_cb(struct urb *urb) { - struct cmd_buf *cmd = (struct cmd_buf *)urb->context; + struct cmd_buf *cmd = urb->context; switch (urb->status) { case 0: @@ -134,7 +134,7 @@ static int hif_usb_send_regout(struct hif_device_usb *hif_dev, static void hif_usb_mgmt_cb(struct urb *urb) { - struct cmd_buf *cmd = (struct cmd_buf *)urb->context; + struct cmd_buf *cmd = urb->context; struct hif_device_usb *hif_dev; unsigned long flags; bool txok = true; @@ -252,7 +252,7 @@ static inline void ath9k_skb_queue_complete(struct hif_device_usb *hif_dev, static void hif_usb_tx_cb(struct urb *urb) { - struct tx_buf *tx_buf = (struct tx_buf *) urb->context; + struct tx_buf *tx_buf = urb->context; struct hif_device_usb *hif_dev; bool txok = true; @@ -687,7 +687,7 @@ static void ath9k_hif_usb_rx_stream(struct hif_device_usb *hif_dev, static void ath9k_hif_usb_rx_cb(struct urb *urb) { - struct rx_buf *rx_buf = (struct rx_buf *)urb->context; + struct rx_buf *rx_buf = urb->context; struct hif_device_usb *hif_dev = rx_buf->hif_dev; struct sk_buff *skb = rx_buf->skb; int ret; @@ -734,7 +734,7 @@ static void ath9k_hif_usb_rx_cb(struct urb *urb) static void ath9k_hif_usb_reg_in_cb(struct urb *urb) { - struct rx_buf *rx_buf = (struct rx_buf *)urb->context; + struct rx_buf *rx_buf = urb->context; struct hif_device_usb *hif_dev = rx_buf->hif_dev; struct sk_buff *skb = rx_buf->skb; int ret; diff --git a/drivers/net/wireless/ath/ath9k/htc_drv_debug.c b/drivers/net/wireless/ath/ath9k/htc_drv_debug.c index 278ddc713fdc25..f7c6d9bc931196 100644 --- a/drivers/net/wireless/ath/ath9k/htc_drv_debug.c +++ b/drivers/net/wireless/ath/ath9k/htc_drv_debug.c @@ -482,7 +482,7 @@ void ath9k_htc_deinit_debug(struct ath9k_htc_priv *priv) int ath9k_htc_init_debug(struct ath_hw *ah) { struct ath_common *common = ath9k_hw_common(ah); - struct ath9k_htc_priv *priv = (struct ath9k_htc_priv *) common->priv; + struct ath9k_htc_priv *priv = common->priv; priv->debug.debugfs_phy = debugfs_create_dir(KBUILD_MODNAME, priv->hw->wiphy->debugfsdir); diff --git a/drivers/net/wireless/ath/ath9k/htc_drv_init.c b/drivers/net/wireless/ath/ath9k/htc_drv_init.c index dae3d9c7b64082..0aa5bdeb44a1b0 100644 --- a/drivers/net/wireless/ath/ath9k/htc_drv_init.c +++ b/drivers/net/wireless/ath/ath9k/htc_drv_init.c @@ -63,12 +63,12 @@ static const struct ieee80211_tpt_blink ath9k_htc_tpt_blink[] = { static void ath9k_htc_op_ps_wakeup(struct ath_common *common) { - ath9k_htc_ps_wakeup((struct ath9k_htc_priv *) common->priv); + ath9k_htc_ps_wakeup(common->priv); } static void ath9k_htc_op_ps_restore(struct ath_common *common) { - ath9k_htc_ps_restore((struct ath9k_htc_priv *) common->priv); + ath9k_htc_ps_restore(common->priv); } static const struct ath_ps_ops ath9k_htc_ps_ops = { @@ -235,7 +235,7 @@ static unsigned int ath9k_regread(void *hw_priv, u32 reg_offset) { struct ath_hw *ah = hw_priv; struct ath_common *common = ath9k_hw_common(ah); - struct ath9k_htc_priv *priv = (struct ath9k_htc_priv *) common->priv; + struct ath9k_htc_priv *priv = common->priv; __be32 val, reg = cpu_to_be32(reg_offset); int r; @@ -257,7 +257,7 @@ static void ath9k_multi_regread(void *hw_priv, u32 *addr, { struct ath_hw *ah = hw_priv; struct ath_common *common = ath9k_hw_common(ah); - struct ath9k_htc_priv *priv = (struct ath9k_htc_priv *) common->priv; + struct ath9k_htc_priv *priv = common->priv; __be32 tmpaddr[8]; __be32 tmpval[8]; int i, ret; @@ -282,7 +282,7 @@ static void ath9k_multi_regread(void *hw_priv, u32 *addr, static void ath9k_regwrite_multi(struct ath_common *common) { - struct ath9k_htc_priv *priv = (struct ath9k_htc_priv *) common->priv; + struct ath9k_htc_priv *priv = common->priv; u32 rsp_status; int r; @@ -303,7 +303,7 @@ static void ath9k_regwrite_single(void *hw_priv, u32 val, u32 reg_offset) { struct ath_hw *ah = hw_priv; struct ath_common *common = ath9k_hw_common(ah); - struct ath9k_htc_priv *priv = (struct ath9k_htc_priv *) common->priv; + struct ath9k_htc_priv *priv = common->priv; const __be32 buf[2] = { cpu_to_be32(reg_offset), cpu_to_be32(val), @@ -324,7 +324,7 @@ static void ath9k_regwrite_buffer(void *hw_priv, u32 val, u32 reg_offset) { struct ath_hw *ah = hw_priv; struct ath_common *common = ath9k_hw_common(ah); - struct ath9k_htc_priv *priv = (struct ath9k_htc_priv *) common->priv; + struct ath9k_htc_priv *priv = common->priv; mutex_lock(&priv->wmi->multi_write_mutex); @@ -347,7 +347,7 @@ static void ath9k_regwrite(void *hw_priv, u32 val, u32 reg_offset) { struct ath_hw *ah = hw_priv; struct ath_common *common = ath9k_hw_common(ah); - struct ath9k_htc_priv *priv = (struct ath9k_htc_priv *) common->priv; + struct ath9k_htc_priv *priv = common->priv; if (atomic_read(&priv->wmi->mwrite_cnt)) ath9k_regwrite_buffer(hw_priv, val, reg_offset); @@ -359,7 +359,7 @@ static void ath9k_enable_regwrite_buffer(void *hw_priv) { struct ath_hw *ah = hw_priv; struct ath_common *common = ath9k_hw_common(ah); - struct ath9k_htc_priv *priv = (struct ath9k_htc_priv *) common->priv; + struct ath9k_htc_priv *priv = common->priv; atomic_inc(&priv->wmi->mwrite_cnt); } @@ -368,7 +368,7 @@ static void ath9k_regwrite_flush(void *hw_priv) { struct ath_hw *ah = hw_priv; struct ath_common *common = ath9k_hw_common(ah); - struct ath9k_htc_priv *priv = (struct ath9k_htc_priv *) common->priv; + struct ath9k_htc_priv *priv = common->priv; atomic_dec(&priv->wmi->mwrite_cnt); @@ -385,7 +385,7 @@ static void ath9k_reg_rmw_buffer(void *hw_priv, { struct ath_hw *ah = hw_priv; struct ath_common *common = ath9k_hw_common(ah); - struct ath9k_htc_priv *priv = (struct ath9k_htc_priv *) common->priv; + struct ath9k_htc_priv *priv = common->priv; u32 rsp_status; int r; @@ -423,7 +423,7 @@ static void ath9k_reg_rmw_flush(void *hw_priv) { struct ath_hw *ah = hw_priv; struct ath_common *common = ath9k_hw_common(ah); - struct ath9k_htc_priv *priv = (struct ath9k_htc_priv *) common->priv; + struct ath9k_htc_priv *priv = common->priv; u32 rsp_status; int r; @@ -455,7 +455,7 @@ static void ath9k_enable_rmw_buffer(void *hw_priv) { struct ath_hw *ah = hw_priv; struct ath_common *common = ath9k_hw_common(ah); - struct ath9k_htc_priv *priv = (struct ath9k_htc_priv *) common->priv; + struct ath9k_htc_priv *priv = common->priv; if (test_bit(HTC_FWFLAG_NO_RMW, &priv->fw_flags)) return; @@ -468,7 +468,7 @@ static void ath9k_reg_rmw_single(void *hw_priv, { struct ath_hw *ah = hw_priv; struct ath_common *common = ath9k_hw_common(ah); - struct ath9k_htc_priv *priv = (struct ath9k_htc_priv *) common->priv; + struct ath9k_htc_priv *priv = common->priv; struct register_rmw buf, buf_ret; int ret; @@ -490,7 +490,7 @@ static u32 ath9k_reg_rmw(void *hw_priv, u32 reg_offset, u32 set, u32 clr) { struct ath_hw *ah = hw_priv; struct ath_common *common = ath9k_hw_common(ah); - struct ath9k_htc_priv *priv = (struct ath9k_htc_priv *) common->priv; + struct ath9k_htc_priv *priv = common->priv; if (test_bit(HTC_FWFLAG_NO_RMW, &priv->fw_flags)) { u32 val; @@ -518,7 +518,7 @@ static void ath_usb_read_cachesize(struct ath_common *common, int *csz) static bool ath_usb_eeprom_read(struct ath_common *common, u32 off, u16 *data) { - struct ath_hw *ah = (struct ath_hw *) common->ah; + struct ath_hw *ah = common->ah; (void)REG_READ(ah, AR5416_EEPROM_OFFSET + (off << AR5416_EEPROM_S)); @@ -970,7 +970,7 @@ int ath9k_htc_probe_device(struct htc_target *htc_handle, struct device *dev, err_init: ath9k_stop_wmi(priv); - hif_dev = (struct hif_device_usb *)htc_handle->hif_dev; + hif_dev = htc_handle->hif_dev; ath9k_hif_usb_dealloc_urbs(hif_dev); ath9k_destroy_wmi(priv); err_free: @@ -988,7 +988,7 @@ void ath9k_htc_disconnect_device(struct htc_target *htc_handle, bool hotunplug) ath9k_deinit_device(htc_handle->drv_priv); ath9k_stop_wmi(htc_handle->drv_priv); - ath9k_hif_usb_dealloc_urbs((struct hif_device_usb *)htc_handle->hif_dev); + ath9k_hif_usb_dealloc_urbs(htc_handle->hif_dev); ath9k_destroy_wmi(htc_handle->drv_priv); ieee80211_free_hw(htc_handle->drv_priv->hw); } diff --git a/drivers/net/wireless/ath/ath9k/htc_drv_txrx.c b/drivers/net/wireless/ath/ath9k/htc_drv_txrx.c index 800177021baffd..efcaeccb055aa5 100644 --- a/drivers/net/wireless/ath/ath9k/htc_drv_txrx.c +++ b/drivers/net/wireless/ath/ath9k/htc_drv_txrx.c @@ -652,9 +652,10 @@ void ath9k_htc_txstatus(struct ath9k_htc_priv *priv, void *wmi_event) struct ath9k_htc_tx_event *tx_pend; int i; - for (i = 0; i < txs->cnt; i++) { - WARN_ON(txs->cnt > HTC_MAX_TX_STATUS); + if (WARN_ON_ONCE(txs->cnt > HTC_MAX_TX_STATUS)) + return; + for (i = 0; i < txs->cnt; i++) { __txs = &txs->txstatus[i]; skb = ath9k_htc_tx_get_packet(priv, __txs); diff --git a/drivers/net/wireless/ath/ath9k/htc_hst.c b/drivers/net/wireless/ath/ath9k/htc_hst.c index 99667aba289df0..eb631fd3336d8d 100644 --- a/drivers/net/wireless/ath/ath9k/htc_hst.c +++ b/drivers/net/wireless/ath/ath9k/htc_hst.c @@ -89,7 +89,7 @@ static void htc_process_target_rdy(struct htc_target *target, void *buf) { struct htc_endpoint *endpoint; - struct htc_ready_msg *htc_ready_msg = (struct htc_ready_msg *) buf; + struct htc_ready_msg *htc_ready_msg = buf; target->credit_size = be16_to_cpu(htc_ready_msg->credit_size); diff --git a/drivers/net/wireless/ath/ath9k/init.c b/drivers/net/wireless/ath/ath9k/init.c index 4f00400c7ffb83..7fad7e75af6a37 100644 --- a/drivers/net/wireless/ath/ath9k/init.c +++ b/drivers/net/wireless/ath/ath9k/init.c @@ -151,12 +151,12 @@ static void ath9k_deinit_softc(struct ath_softc *sc); static void ath9k_op_ps_wakeup(struct ath_common *common) { - ath9k_ps_wakeup((struct ath_softc *) common->priv); + ath9k_ps_wakeup(common->priv); } static void ath9k_op_ps_restore(struct ath_common *common) { - ath9k_ps_restore((struct ath_softc *) common->priv); + ath9k_ps_restore(common->priv); } static const struct ath_ps_ops ath9k_ps_ops = { @@ -174,7 +174,7 @@ static void ath9k_iowrite32(void *hw_priv, u32 val, u32 reg_offset) { struct ath_hw *ah = hw_priv; struct ath_common *common = ath9k_hw_common(ah); - struct ath_softc *sc = (struct ath_softc *) common->priv; + struct ath_softc *sc = common->priv; if (NR_CPUS > 1 && ah->config.serialize_regmode == SER_REG_MODE_ON) { unsigned long flags; @@ -189,7 +189,7 @@ static unsigned int ath9k_ioread32(void *hw_priv, u32 reg_offset) { struct ath_hw *ah = hw_priv; struct ath_common *common = ath9k_hw_common(ah); - struct ath_softc *sc = (struct ath_softc *) common->priv; + struct ath_softc *sc = common->priv; u32 val; if (NR_CPUS > 1 && ah->config.serialize_regmode == SER_REG_MODE_ON) { @@ -229,7 +229,7 @@ static unsigned int ath9k_reg_rmw(void *hw_priv, u32 reg_offset, u32 set, u32 cl { struct ath_hw *ah = hw_priv; struct ath_common *common = ath9k_hw_common(ah); - struct ath_softc *sc = (struct ath_softc *) common->priv; + struct ath_softc *sc = common->priv; unsigned long flags; u32 val; @@ -608,7 +608,7 @@ static int ath9k_nvmem_request_eeprom(struct ath_softc *sc) } /* devres manages the calibration values release on shutdown */ - ah->nvmem_blob = (u16 *)devm_kmemdup(sc->dev, buf, len, GFP_KERNEL); + ah->nvmem_blob = devm_kmemdup(sc->dev, buf, len, GFP_KERNEL); kfree(buf); if (!ah->nvmem_blob) return -ENOMEM; diff --git a/drivers/net/wireless/ath/ath9k/link.c b/drivers/net/wireless/ath/ath9k/link.c index 9d84003db800e9..d1e5767aab3cbc 100644 --- a/drivers/net/wireless/ath/ath9k/link.c +++ b/drivers/net/wireless/ath/ath9k/link.c @@ -304,7 +304,7 @@ void ath_paprd_calibrate(struct work_struct *work) void ath_ani_calibrate(struct timer_list *t) { struct ath_common *common = from_timer(common, t, ani.timer); - struct ath_softc *sc = (struct ath_softc *)common->priv; + struct ath_softc *sc = common->priv; struct ath_hw *ah = sc->sc_ah; bool longcal = false; bool shortcal = false; diff --git a/drivers/net/wireless/ath/ath9k/main.c b/drivers/net/wireless/ath/ath9k/main.c index 1494feedb27dbc..c48ff0ffbfefb3 100644 --- a/drivers/net/wireless/ath/ath9k/main.c +++ b/drivers/net/wireless/ath/ath9k/main.c @@ -2383,7 +2383,22 @@ static void ath9k_sw_scan_start(struct ieee80211_hw *hw, { struct ath_softc *sc = hw->priv; struct ath_common *common = ath9k_hw_common(sc->sc_ah); + struct cfg80211_chan_def *chandef = &sc->cur_chan->chandef; + struct ieee80211_channel *chan = chandef->chan; + int pos = chan->hw_value; set_bit(ATH_OP_SCANNING, &common->op_flags); + + /* Reset current survey */ + if (!sc->cur_chan->offchannel) { + if (sc->cur_survey != &sc->survey[pos]) { + if (sc->cur_survey) + sc->cur_survey->filled &= ~SURVEY_INFO_IN_USE; + sc->cur_survey = &sc->survey[pos]; + } + + memset(sc->cur_survey, 0, sizeof(struct survey_info)); + sc->cur_survey->filled |= SURVEY_INFO_IN_USE; + } } static void ath9k_sw_scan_complete(struct ieee80211_hw *hw, diff --git a/drivers/net/wireless/ath/ath9k/pci.c b/drivers/net/wireless/ath/ath9k/pci.c index 0633589b85c23a..e655cd8bbf94e1 100644 --- a/drivers/net/wireless/ath/ath9k/pci.c +++ b/drivers/net/wireless/ath/ath9k/pci.c @@ -781,7 +781,7 @@ static const struct pci_device_id ath_pci_id_table[] = { /* return bus cachesize in 4B word units */ static void ath_pci_read_cachesize(struct ath_common *common, int *csz) { - struct ath_softc *sc = (struct ath_softc *) common->priv; + struct ath_softc *sc = common->priv; u8 u8tmp; pci_read_config_byte(to_pci_dev(sc->dev), PCI_CACHE_LINE_SIZE, &u8tmp); @@ -799,7 +799,7 @@ static void ath_pci_read_cachesize(struct ath_common *common, int *csz) static bool ath_pci_eeprom_read(struct ath_common *common, u32 off, u16 *data) { - struct ath_hw *ah = (struct ath_hw *) common->ah; + struct ath_hw *ah = common->ah; common->ops->read(ah, AR5416_EEPROM_OFFSET + (off << AR5416_EEPROM_S)); @@ -820,7 +820,7 @@ static bool ath_pci_eeprom_read(struct ath_common *common, u32 off, u16 *data) /* Need to be called after we discover btcoex capabilities */ static void ath_pci_aspm_init(struct ath_common *common) { - struct ath_softc *sc = (struct ath_softc *) common->priv; + struct ath_softc *sc = common->priv; struct ath_hw *ah = sc->sc_ah; struct pci_dev *pdev = to_pci_dev(sc->dev); struct pci_dev *parent; diff --git a/drivers/net/wireless/ath/wcn36xx/main.c b/drivers/net/wireless/ath/wcn36xx/main.c index 2bd1163177f08f..41119fb177e306 100644 --- a/drivers/net/wireless/ath/wcn36xx/main.c +++ b/drivers/net/wireless/ath/wcn36xx/main.c @@ -1644,7 +1644,7 @@ static int wcn36xx_probe(struct platform_device *pdev) return ret; } -static int wcn36xx_remove(struct platform_device *pdev) +static void wcn36xx_remove(struct platform_device *pdev) { struct ieee80211_hw *hw = platform_get_drvdata(pdev); struct wcn36xx *wcn = hw->priv; @@ -1666,8 +1666,6 @@ static int wcn36xx_remove(struct platform_device *pdev) mutex_destroy(&wcn->hal_mutex); ieee80211_free_hw(hw); - - return 0; } static const struct of_device_id wcn36xx_of_match[] = { @@ -1678,7 +1676,7 @@ MODULE_DEVICE_TABLE(of, wcn36xx_of_match); static struct platform_driver wcn36xx_driver = { .probe = wcn36xx_probe, - .remove = wcn36xx_remove, + .remove_new = wcn36xx_remove, .driver = { .name = "wcn36xx", .of_match_table = wcn36xx_of_match, diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c index a194b0e68eb53a..b6d458e022fad3 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c @@ -578,18 +578,16 @@ static int __init brcmf_common_pd_probe(struct platform_device *pdev) return 0; } -static int brcmf_common_pd_remove(struct platform_device *pdev) +static void brcmf_common_pd_remove(struct platform_device *pdev) { brcmf_dbg(INFO, "Enter\n"); if (brcmfmac_pdata->power_off) brcmfmac_pdata->power_off(); - - return 0; } static struct platform_driver brcmf_pd = { - .remove = brcmf_common_pd_remove, + .remove_new = brcmf_common_pd_remove, .driver = { .name = BRCMFMAC_PDATA_NAME, } diff --git a/drivers/net/wireless/intel/iwlegacy/common.c b/drivers/net/wireless/intel/iwlegacy/common.c index 054fef680aba0e..17570d62c89618 100644 --- a/drivers/net/wireless/intel/iwlegacy/common.c +++ b/drivers/net/wireless/intel/iwlegacy/common.c @@ -541,6 +541,9 @@ il_leds_init(struct il_priv *il) il->led.name = kasprintf(GFP_KERNEL, "%s-led", wiphy_name(il->hw->wiphy)); + if (!il->led.name) + return; + il->led.brightness_set = il_led_brightness_set; il->led.blink_set = il_led_blink_set; il->led.max_brightness = 1; diff --git a/drivers/net/wireless/intel/iwlwifi/fw/dbg.c b/drivers/net/wireless/intel/iwlwifi/fw/dbg.c index 3975a53a9f2099..f6e399d1e95c6b 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/dbg.c +++ b/drivers/net/wireless/intel/iwlwifi/fw/dbg.c @@ -880,10 +880,10 @@ iwl_fw_error_dump_file(struct iwl_fw_runtime *fwrt, cpu_to_le32(fwrt->trans->hw_rev_step); memcpy(dump_info->fw_human_readable, fwrt->fw->human_readable, sizeof(dump_info->fw_human_readable)); - strncpy(dump_info->dev_human_readable, fwrt->trans->name, - sizeof(dump_info->dev_human_readable) - 1); - strncpy(dump_info->bus_human_readable, fwrt->dev->bus->name, - sizeof(dump_info->bus_human_readable) - 1); + strscpy_pad(dump_info->dev_human_readable, fwrt->trans->name, + sizeof(dump_info->dev_human_readable)); + strscpy_pad(dump_info->bus_human_readable, fwrt->dev->bus->name, + sizeof(dump_info->bus_human_readable)); dump_info->num_of_lmacs = fwrt->smem_cfg.num_lmacs; dump_info->lmac_err_id[0] = cpu_to_le32(fwrt->dump.lmac_err_id[0]); @@ -3395,3 +3395,13 @@ void iwl_fw_disable_dbg_asserts(struct iwl_fw_runtime *fwrt) iwl_trans_send_cmd(fwrt->trans, &hcmd); } IWL_EXPORT_SYMBOL(iwl_fw_disable_dbg_asserts); + +void iwl_fw_dbg_clear_monitor_buf(struct iwl_fw_runtime *fwrt) +{ + struct iwl_fw_dbg_params params = {0}; + + iwl_fw_dbg_stop_sync(fwrt); + iwl_dbg_tlv_init_cfg(fwrt); + iwl_fw_dbg_stop_restart_recording(fwrt, ¶ms, false); +} +IWL_EXPORT_SYMBOL(iwl_fw_dbg_clear_monitor_buf); diff --git a/drivers/net/wireless/intel/iwlwifi/fw/dbg.h b/drivers/net/wireless/intel/iwlwifi/fw/dbg.h index 66b233250c7c3b..eb38c686b5cb57 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/dbg.h +++ b/drivers/net/wireless/intel/iwlwifi/fw/dbg.h @@ -330,6 +330,7 @@ void iwl_send_dbg_dump_complete_cmd(struct iwl_fw_runtime *fwrt, u32 timepoint, u32 timepoint_data); void iwl_fw_disable_dbg_asserts(struct iwl_fw_runtime *fwrt); +void iwl_fw_dbg_clear_monitor_buf(struct iwl_fw_runtime *fwrt); #define IWL_FW_CHECK_FAILED(_obj, _fmt, ...) \ IWL_ERR_LIMIT(_obj, _fmt, __VA_ARGS__) diff --git a/drivers/net/wireless/intel/iwlwifi/fw/file.h b/drivers/net/wireless/intel/iwlwifi/fw/file.h index 03f6e520145f0b..c38e5194c55f0d 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/file.h +++ b/drivers/net/wireless/intel/iwlwifi/fw/file.h @@ -20,7 +20,7 @@ struct iwl_ucode_header { __le32 init_size; /* bytes of init code */ __le32 init_data_size; /* bytes of init data */ __le32 boot_size; /* bytes of bootstrap code */ - u8 data[0]; /* in same order as sizes */ + u8 data[]; /* in same order as sizes */ } v1; struct { __le32 build; /* build number */ @@ -29,7 +29,7 @@ struct iwl_ucode_header { __le32 init_size; /* bytes of init code */ __le32 init_data_size; /* bytes of init data */ __le32 boot_size; /* bytes of bootstrap code */ - u8 data[0]; /* in same order as sizes */ + u8 data[]; /* in same order as sizes */ } v2; } u; }; @@ -243,6 +243,8 @@ typedef unsigned int __bitwise iwl_ucode_tlv_api_t; * version tables. * @IWL_UCODE_TLV_API_REDUCED_SCAN_CONFIG: This ucode supports v3 of * SCAN_CONFIG_DB_CMD_API_S. + * @IWL_UCODE_TLV_API_NO_HOST_DISABLE_TX: Firmware offloaded the station disable tx + * logic. * * @NUM_IWL_UCODE_TLV_API: number of bits used */ @@ -280,6 +282,7 @@ enum iwl_ucode_tlv_api { IWL_UCODE_TLV_API_ADWELL_HB_DEF_N_AP = (__force iwl_ucode_tlv_api_t)57, IWL_UCODE_TLV_API_SCAN_EXT_CHAN_VER = (__force iwl_ucode_tlv_api_t)58, IWL_UCODE_TLV_API_BAND_IN_RX_DATA = (__force iwl_ucode_tlv_api_t)59, + IWL_UCODE_TLV_API_NO_HOST_DISABLE_TX = (__force iwl_ucode_tlv_api_t)66, NUM_IWL_UCODE_TLV_API /* diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c b/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c index b658cf228fbe2d..3b14f647674350 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c +++ b/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c @@ -1274,7 +1274,7 @@ iwl_dbg_tlv_tp_trigger(struct iwl_fw_runtime *fwrt, bool sync, return 0; } -static void iwl_dbg_tlv_init_cfg(struct iwl_fw_runtime *fwrt) +void iwl_dbg_tlv_init_cfg(struct iwl_fw_runtime *fwrt) { enum iwl_fw_ini_buffer_location *ini_dest = &fwrt->trans->dbg.ini_dest; int ret, i; diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.h b/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.h index 06fb7d6653905a..7ed6329fd8ca1f 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.h +++ b/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.h @@ -57,6 +57,7 @@ void _iwl_dbg_tlv_time_point(struct iwl_fw_runtime *fwrt, enum iwl_fw_ini_time_point tp_id, union iwl_dbg_tlv_tp_data *tp_data, bool sync); +void iwl_dbg_tlv_init_cfg(struct iwl_fw_runtime *fwrt); static inline void iwl_dbg_tlv_time_point(struct iwl_fw_runtime *fwrt, enum iwl_fw_ini_time_point tp_id, diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-devtrace-data.h b/drivers/net/wireless/intel/iwlwifi/iwl-devtrace-data.h index 347fd95c4e3a5d..2c280a2fe3df32 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-devtrace-data.h +++ b/drivers/net/wireless/intel/iwlwifi/iwl-devtrace-data.h @@ -3,7 +3,7 @@ * * Copyright(c) 2009 - 2014 Intel Corporation. All rights reserved. * Copyright(c) 2015 Intel Deutschland GmbH - * Copyright(c) 2018 - 2019 Intel Corporation + * Copyright(c) 2018 - 2019, 2023 Intel Corporation *****************************************************************************/ #if !defined(__IWLWIFI_DEVICE_TRACE_DATA) || defined(TRACE_HEADER_MULTI_READ) @@ -36,20 +36,17 @@ TRACE_EVENT(iwlwifi_dev_tx_tb, TRACE_EVENT(iwlwifi_dev_rx_data, TP_PROTO(const struct device *dev, - const struct iwl_trans *trans, - void *rxbuf, size_t len), - TP_ARGS(dev, trans, rxbuf, len), + void *rxbuf, size_t len, size_t start), + TP_ARGS(dev, rxbuf, len, start), TP_STRUCT__entry( DEV_ENTRY - __dynamic_array(u8, data, - len - iwl_rx_trace_len(trans, rxbuf, len, NULL)) + __dynamic_array(u8, data, len - start) ), TP_fast_assign( - size_t offs = iwl_rx_trace_len(trans, rxbuf, len, NULL); DEV_ASSIGN; - if (offs < len) + if (start < len) memcpy(__get_dynamic_array(data), - ((u8 *)rxbuf) + offs, len - offs); + ((u8 *)rxbuf) + start, len - start); ), TP_printk("[%s] RX frame data", __get_str(dev)) ); diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-devtrace-iwlwifi.h b/drivers/net/wireless/intel/iwlwifi/iwl-devtrace-iwlwifi.h index 46ed723f138a66..e656bf6bc00338 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-devtrace-iwlwifi.h +++ b/drivers/net/wireless/intel/iwlwifi/iwl-devtrace-iwlwifi.h @@ -4,7 +4,7 @@ * Copyright(c) 2009 - 2014 Intel Corporation. All rights reserved. * Copyright(c) 2015 Intel Mobile Communications GmbH * Copyright(c) 2016 - 2017 Intel Deutschland GmbH - * Copyright(c) 2018 Intel Corporation + * Copyright(c) 2018, 2023 Intel Corporation *****************************************************************************/ #if !defined(__IWLWIFI_DEVICE_TRACE_IWLWIFI) || defined(TRACE_HEADER_MULTI_READ) @@ -50,23 +50,20 @@ TRACE_EVENT(iwlwifi_dev_hcmd, ); TRACE_EVENT(iwlwifi_dev_rx, - TP_PROTO(const struct device *dev, const struct iwl_trans *trans, - struct iwl_rx_packet *pkt, size_t len), - TP_ARGS(dev, trans, pkt, len), + TP_PROTO(const struct device *dev, + struct iwl_rx_packet *pkt, size_t len, size_t trace_len, + size_t hdr_offset), + TP_ARGS(dev, pkt, len, trace_len, hdr_offset), TP_STRUCT__entry( DEV_ENTRY __field(u16, cmd) __field(u8, hdr_offset) - __dynamic_array(u8, rxbuf, - iwl_rx_trace_len(trans, pkt, len, NULL)) + __dynamic_array(u8, rxbuf, trace_len) ), TP_fast_assign( - size_t hdr_offset = 0; - DEV_ASSIGN; __entry->cmd = WIDE_ID(pkt->hdr.group_id, pkt->hdr.cmd); - memcpy(__get_dynamic_array(rxbuf), pkt, - iwl_rx_trace_len(trans, pkt, len, &hdr_offset)); + memcpy(__get_dynamic_array(rxbuf), pkt, trace_len); __entry->hdr_offset = hdr_offset; ), TP_printk("[%s] RX cmd %#.2x", diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-devtrace.c b/drivers/net/wireless/intel/iwlwifi/iwl-devtrace.c index e46639b097f461..7e686297963d66 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-devtrace.c +++ b/drivers/net/wireless/intel/iwlwifi/iwl-devtrace.c @@ -2,7 +2,7 @@ /****************************************************************************** * * Copyright(c) 2009 - 2014 Intel Corporation. All rights reserved. - * Copyright (C) 2018 Intel Corporation + * Copyright (C) 2018, 2023 Intel Corporation *****************************************************************************/ #include @@ -20,4 +20,17 @@ EXPORT_TRACEPOINT_SYMBOL(iwlwifi_dev_ucode_event); EXPORT_TRACEPOINT_SYMBOL(iwlwifi_dev_ucode_cont_event); EXPORT_TRACEPOINT_SYMBOL(iwlwifi_dev_ucode_wrap_event); -#endif +#else +#include "iwl-devtrace.h" +#endif /* __CHECKER__ */ + +void __trace_iwlwifi_dev_rx(struct iwl_trans *trans, void *pkt, size_t len) +{ + size_t hdr_offset = 0, trace_len; + + trace_len = iwl_rx_trace_len(trans, pkt, len, &hdr_offset); + trace_iwlwifi_dev_rx(trans->dev, pkt, len, trace_len, hdr_offset); + + if (trace_len < len) + trace_iwlwifi_dev_rx_data(trans->dev, pkt, len, trace_len); +} diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-devtrace.h b/drivers/net/wireless/intel/iwlwifi/iwl-devtrace.h index 01fb7b900a6d34..c3e09f4fefeb9c 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-devtrace.h +++ b/drivers/net/wireless/intel/iwlwifi/iwl-devtrace.h @@ -7,12 +7,12 @@ *****************************************************************************/ #ifndef __IWLWIFI_DEVICE_TRACE +#define __IWLWIFI_DEVICE_TRACE #include #include #include #include #include "iwl-trans.h" -#if !defined(__IWLWIFI_DEVICE_TRACE) static inline bool iwl_trace_data(struct sk_buff *skb) { struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); @@ -70,9 +70,6 @@ static inline size_t iwl_rx_trace_len(const struct iwl_trans *trans, return sizeof(__le32) + sizeof(*cmd) + trans->rx_mpdu_cmd_hdr_size + ieee80211_hdrlen(hdr->frame_control); } -#endif - -#define __IWLWIFI_DEVICE_TRACE #include #include @@ -98,4 +95,20 @@ static inline void trace_ ## name(proto) {} #include "iwl-devtrace-data.h" #include "iwl-devtrace-iwlwifi.h" +#ifdef CONFIG_IWLWIFI_DEVICE_TRACING +DECLARE_TRACEPOINT(iwlwifi_dev_rx); +DECLARE_TRACEPOINT(iwlwifi_dev_rx_data); +#endif + +void __trace_iwlwifi_dev_rx(struct iwl_trans *trans, void *pkt, size_t len); + +static inline void maybe_trace_iwlwifi_dev_rx(struct iwl_trans *trans, + void *pkt, size_t len) +{ +#ifdef CONFIG_IWLWIFI_DEVICE_TRACING + if (tracepoint_enabled(iwlwifi_dev_rx) || + tracepoint_enabled(iwlwifi_dev_rx_data)) + __trace_iwlwifi_dev_rx(trans, pkt, len); +#endif +} #endif /* __IWLWIFI_DEVICE_TRACE */ diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c index 6015e1255d2aca..480f8edbfd35df 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c +++ b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c @@ -1029,7 +1029,8 @@ iwl_nvm_fixup_sband_iftd(struct iwl_trans *trans, IEEE80211_EHT_PHY_CAP3_NG_16_MU_FEEDBACK | IEEE80211_EHT_PHY_CAP3_CODEBOOK_4_2_SU_FDBK | IEEE80211_EHT_PHY_CAP3_CODEBOOK_7_5_MU_FDBK | - IEEE80211_EHT_PHY_CAP3_TRIG_MU_BF_PART_BW_FDBK); + IEEE80211_EHT_PHY_CAP3_TRIG_MU_BF_PART_BW_FDBK | + IEEE80211_EHT_PHY_CAP3_TRIG_CQI_FDBK); iftype_data->eht_cap.eht_cap_elem.phy_cap_info[4] &= ~(IEEE80211_EHT_PHY_CAP4_PART_BW_DL_MU_MIMO | IEEE80211_EHT_PHY_CAP4_POWER_BOOST_FACT_SUPP); diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-op-mode.h b/drivers/net/wireless/intel/iwlwifi/iwl-op-mode.h index af5f9b210f2258..3dc618a7c70feb 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-op-mode.h +++ b/drivers/net/wireless/intel/iwlwifi/iwl-op-mode.h @@ -64,8 +64,6 @@ struct iwl_cfg; * received on the RSS queue(s). The queue parameter indicates which of the * RSS queues received this frame; it will always be non-zero. * This method must not sleep. - * @async_cb: called when an ASYNC command with CMD_WANT_ASYNC_CALLBACK set - * completes. Must be atomic. * @queue_full: notifies that a HW queue is full. * Must be atomic and called with BH disabled. * @queue_not_full: notifies that a HW queue is not full any more. @@ -96,8 +94,6 @@ struct iwl_op_mode_ops { struct iwl_rx_cmd_buffer *rxb); void (*rx_rss)(struct iwl_op_mode *op_mode, struct napi_struct *napi, struct iwl_rx_cmd_buffer *rxb, unsigned int queue); - void (*async_cb)(struct iwl_op_mode *op_mode, - const struct iwl_device_cmd *cmd); void (*queue_full)(struct iwl_op_mode *op_mode, int queue); void (*queue_not_full)(struct iwl_op_mode *op_mode, int queue); bool (*hw_rf_kill)(struct iwl_op_mode *op_mode, bool state); @@ -147,13 +143,6 @@ static inline void iwl_op_mode_rx_rss(struct iwl_op_mode *op_mode, op_mode->ops->rx_rss(op_mode, napi, rxb, queue); } -static inline void iwl_op_mode_async_cb(struct iwl_op_mode *op_mode, - const struct iwl_device_cmd *cmd) -{ - if (op_mode->ops->async_cb) - op_mode->ops->async_cb(op_mode, cmd); -} - static inline void iwl_op_mode_queue_full(struct iwl_op_mode *op_mode, int queue) { diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-trans.c b/drivers/net/wireless/intel/iwlwifi/iwl-trans.c index 4bd759432d44b0..f95098c21c7d59 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-trans.c +++ b/drivers/net/wireless/intel/iwlwifi/iwl-trans.c @@ -172,10 +172,6 @@ int iwl_trans_send_cmd(struct iwl_trans *trans, struct iwl_host_cmd *cmd) return -EIO; } - if (WARN_ON((cmd->flags & CMD_WANT_ASYNC_CALLBACK) && - !(cmd->flags & CMD_ASYNC))) - return -EINVAL; - if (!(cmd->flags & CMD_ASYNC)) lock_map_acquire_read(&trans->sync_cmd_lockdep_map); diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-trans.h b/drivers/net/wireless/intel/iwlwifi/iwl-trans.h index 05e72a2125b3c6..ef7dc0a7b56cb9 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-trans.h +++ b/drivers/net/wireless/intel/iwlwifi/iwl-trans.h @@ -110,8 +110,7 @@ static inline u32 iwl_rx_packet_payload_len(const struct iwl_rx_packet *pkt) * @CMD_WANT_SKB: Not valid with CMD_ASYNC. The caller needs the buffer of * the response. The caller needs to call iwl_free_resp when done. * @CMD_SEND_IN_RFKILL: Send the command even if the NIC is in RF-kill. - * @CMD_WANT_ASYNC_CALLBACK: the op_mode's async callback function must be - * called after this command completes. Valid only with CMD_ASYNC. + * @CMD_BLOCK_TXQS: Block TXQs while the comment is executing. * @CMD_SEND_IN_D3: Allow the command to be sent in D3 mode, relevant to * SUSPEND and RESUME commands. We are in D3 mode when we set * trans->system_pm_mode to IWL_PLAT_PM_MODE_D3. @@ -120,7 +119,7 @@ enum CMD_MODE { CMD_ASYNC = BIT(0), CMD_WANT_SKB = BIT(1), CMD_SEND_IN_RFKILL = BIT(2), - CMD_WANT_ASYNC_CALLBACK = BIT(3), + CMD_BLOCK_TXQS = BIT(3), CMD_SEND_IN_D3 = BIT(4), }; @@ -534,11 +533,6 @@ struct iwl_pnvm_image { * @wait_txq_empty: wait until specific tx queue is empty. May sleep. * @freeze_txq_timer: prevents the timer of the queue from firing until the * queue is set to awake. Must be atomic. - * @block_txq_ptrs: stop updating the write pointers of the Tx queues. Note - * that the transport needs to refcount the calls since this function - * will be called several times with block = true, and then the queues - * need to be unblocked only after the same number of calls with - * block = false. * @write8: write a u8 to a register at offset ofs from the BAR * @write32: write a u32 to a register at offset ofs from the BAR * @read32: read a u32 register at offset ofs from the BAR @@ -613,7 +607,6 @@ struct iwl_trans_ops { int (*wait_txq_empty)(struct iwl_trans *trans, int queue); void (*freeze_txq_timer)(struct iwl_trans *trans, unsigned long txqs, bool freeze); - void (*block_txq_ptrs)(struct iwl_trans *trans, bool block); void (*write8)(struct iwl_trans *trans, u32 ofs, u8 val); void (*write32)(struct iwl_trans *trans, u32 ofs, u32 val); @@ -1407,18 +1400,6 @@ static inline void iwl_trans_freeze_txq_timer(struct iwl_trans *trans, trans->ops->freeze_txq_timer(trans, txqs, freeze); } -static inline void iwl_trans_block_txq_ptrs(struct iwl_trans *trans, - bool block) -{ - if (WARN_ON_ONCE(trans->state != IWL_TRANS_FW_ALIVE)) { - IWL_ERR(trans, "%s bad state = %d\n", __func__, trans->state); - return; - } - - if (trans->ops->block_txq_ptrs) - trans->ops->block_txq_ptrs(trans, block); -} - static inline int iwl_trans_wait_tx_queues_empty(struct iwl_trans *trans, u32 txqs) { diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c b/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c index 329c545f65fde8..e016fce7ab2464 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c @@ -1714,6 +1714,20 @@ static ssize_t iwl_dbgfs_fw_dbg_collect_write(struct iwl_mvm *mvm, return count; } +static ssize_t iwl_dbgfs_fw_dbg_clear_write(struct iwl_mvm *mvm, + char *buf, size_t count, + loff_t *ppos) +{ + if (mvm->trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_9000) + return -EOPNOTSUPP; + + mutex_lock(&mvm->mutex); + iwl_fw_dbg_clear_monitor_buf(&mvm->fwrt); + mutex_unlock(&mvm->mutex); + + return count; +} + static ssize_t iwl_dbgfs_dbg_time_point_write(struct iwl_mvm *mvm, char *buf, size_t count, loff_t *ppos) @@ -2166,6 +2180,7 @@ MVM_DEBUGFS_WRITE_FILE_OPS(bt_force_ant, 10); MVM_DEBUGFS_READ_WRITE_FILE_OPS(scan_ant_rxchain, 8); MVM_DEBUGFS_READ_WRITE_FILE_OPS(fw_dbg_conf, 8); MVM_DEBUGFS_WRITE_FILE_OPS(fw_dbg_collect, 64); +MVM_DEBUGFS_WRITE_FILE_OPS(fw_dbg_clear, 64); MVM_DEBUGFS_WRITE_FILE_OPS(dbg_time_point, 64); MVM_DEBUGFS_WRITE_FILE_OPS(indirection_tbl, (IWL_RSS_INDIRECTION_TABLE_SIZE * 2)); @@ -2372,6 +2387,7 @@ void iwl_mvm_dbgfs_register(struct iwl_mvm *mvm) MVM_DEBUGFS_ADD_FILE(prph_reg, mvm->debugfs_dir, 0600); MVM_DEBUGFS_ADD_FILE(fw_dbg_conf, mvm->debugfs_dir, 0600); MVM_DEBUGFS_ADD_FILE(fw_dbg_collect, mvm->debugfs_dir, 0200); + MVM_DEBUGFS_ADD_FILE(fw_dbg_clear, mvm->debugfs_dir, 0200); MVM_DEBUGFS_ADD_FILE(dbg_time_point, mvm->debugfs_dir, 0200); MVM_DEBUGFS_ADD_FILE(send_echo_cmd, mvm->debugfs_dir, 0200); MVM_DEBUGFS_ADD_FILE(indirection_tbl, mvm->debugfs_dir, 0200); diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mld-sta.c b/drivers/net/wireless/intel/iwlwifi/mvm/mld-sta.c index 6af606e5da657e..1628bf55458fcb 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/mld-sta.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/mld-sta.c @@ -874,6 +874,9 @@ void iwl_mvm_mld_sta_modify_disable_tx(struct iwl_mvm *mvm, cmd.sta_id = cpu_to_le32(mvmsta->deflink.sta_id); cmd.disable = cpu_to_le32(disable); + if (WARN_ON(iwl_mvm_has_no_host_disable_tx(mvm))) + return; + ret = iwl_mvm_send_cmd_pdu(mvm, WIDE_ID(MAC_CONF_GROUP, STA_DISABLE_TX_CMD), CMD_ASYNC, sizeof(cmd), &cmd); diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h index f2af3e57140906..40627961b834a2 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h +++ b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h @@ -947,6 +947,7 @@ struct iwl_mvm { /* the vif that requested the current scan */ struct iwl_mvm_vif *scan_vif; + u8 scan_link_id; /* rx chain antennas set through debugfs for the scan command */ u8 scan_rx_ant; @@ -1513,6 +1514,12 @@ static inline bool iwl_mvm_has_quota_low_latency(struct iwl_mvm *mvm) IWL_UCODE_TLV_API_QUOTA_LOW_LATENCY); } +static inline bool iwl_mvm_has_no_host_disable_tx(struct iwl_mvm *mvm) +{ + return fw_has_api(&mvm->fw->ucode_capa, + IWL_UCODE_TLV_API_NO_HOST_DISABLE_TX); +} + static inline bool iwl_mvm_has_tlc_offload(const struct iwl_mvm *mvm) { return fw_has_capa(&mvm->fw->ucode_capa, diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c index 1627b2f819db1d..adbbe19aeae5dc 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c @@ -1703,18 +1703,6 @@ void iwl_mvm_rx_mq(struct iwl_op_mode *op_mode, iwl_mvm_rx_common(mvm, rxb, pkt); } -static void iwl_mvm_async_cb(struct iwl_op_mode *op_mode, - const struct iwl_device_cmd *cmd) -{ - struct iwl_mvm *mvm = IWL_OP_MODE_GET_MVM(op_mode); - - /* - * For now, we only set the CMD_WANT_ASYNC_CALLBACK for ADD_STA - * commands that need to block the Tx queues. - */ - iwl_trans_block_txq_ptrs(mvm->trans, false); -} - static int iwl_mvm_is_static_queue(struct iwl_mvm *mvm, int queue) { return queue == mvm->aux_queue || queue == mvm->probe_queue || @@ -2024,7 +2012,6 @@ static void iwl_op_mode_mvm_time_point(struct iwl_op_mode *op_mode, #define IWL_MVM_COMMON_OPS \ /* these could be differentiated */ \ - .async_cb = iwl_mvm_async_cb, \ .queue_full = iwl_mvm_stop_sw_queue, \ .queue_not_full = iwl_mvm_wake_sw_queue, \ .hw_rf_kill = iwl_mvm_set_hw_rfkill_state, \ diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/scan.c b/drivers/net/wireless/intel/iwlwifi/mvm/scan.c index 75c5c58e14a516..7b6f1cdca067e8 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/scan.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/scan.c @@ -101,6 +101,7 @@ struct iwl_mvm_scan_params { bool scan_6ghz; bool enable_6ghz_passive; bool respect_p2p_go, respect_p2p_go_hb; + s8 tsf_report_link_id; u8 bssid[ETH_ALEN] __aligned(2); }; @@ -2342,20 +2343,15 @@ iwl_mvm_scan_umac_fill_general_p_v12(struct iwl_mvm *mvm, if (gen_flags & IWL_UMAC_SCAN_GEN_FLAGS_V2_FRAGMENTED_LMAC2) gp->num_of_fragments[SCAN_HB_LMAC_IDX] = IWL_SCAN_NUM_OF_FRAGS; + mvm->scan_link_id = 0; + if (version < 16) { gp->scan_start_mac_or_link_id = scan_vif->id; } else { - struct iwl_mvm_vif_link_info *link_info; - u8 link_id = 0; + struct iwl_mvm_vif_link_info *link_info = + scan_vif->link[params->tsf_report_link_id]; - /* Use one of the active link (if any). In the future it would - * be possible that the link ID would be part of the scan - * request coming from upper layers so we would need to use it. - */ - if (vif->active_links) - link_id = ffs(vif->active_links) - 1; - - link_info = scan_vif->link[link_id]; + mvm->scan_link_id = params->tsf_report_link_id; if (!WARN_ON(!link_info)) gp->scan_start_mac_or_link_id = link_info->fw_link_id; } @@ -2977,6 +2973,14 @@ int iwl_mvm_reg_scan_start(struct iwl_mvm *mvm, struct ieee80211_vif *vif, if (req->duration) params.iter_notif = true; + params.tsf_report_link_id = req->tsf_report_link_id; + if (params.tsf_report_link_id < 0) { + if (vif->active_links) + params.tsf_report_link_id = __ffs(vif->active_links); + else + params.tsf_report_link_id = 0; + } + iwl_mvm_build_scan_probe(mvm, vif, ies, ¶ms); iwl_mvm_scan_6ghz_passive_scan(mvm, ¶ms, vif); @@ -3164,8 +3168,13 @@ void iwl_mvm_rx_umac_scan_complete_notif(struct iwl_mvm *mvm, .aborted = aborted, .scan_start_tsf = mvm->scan_start, }; + struct iwl_mvm_vif *scan_vif = mvm->scan_vif; + struct iwl_mvm_vif_link_info *link_info = + scan_vif->link[mvm->scan_link_id]; + + if (!WARN_ON(!link_info)) + memcpy(info.tsf_bssid, link_info->bssid, ETH_ALEN); - memcpy(info.tsf_bssid, mvm->scan_vif->deflink.bssid, ETH_ALEN); ieee80211_scan_completed(mvm->hw, &info); mvm->scan_vif = NULL; cancel_delayed_work(&mvm->scan_timeout_dwork); diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/sta.c b/drivers/net/wireless/intel/iwlwifi/mvm/sta.c index bba96a96889066..efe3e111ea0a78 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/sta.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/sta.c @@ -4111,10 +4111,8 @@ void iwl_mvm_sta_modify_sleep_tx_count(struct iwl_mvm *mvm, } /* block the Tx queues until the FW updated the sleep Tx count */ - iwl_trans_block_txq_ptrs(mvm->trans, true); - ret = iwl_mvm_send_cmd_pdu(mvm, ADD_STA, - CMD_ASYNC | CMD_WANT_ASYNC_CALLBACK, + CMD_ASYNC | CMD_BLOCK_TXQS, iwl_mvm_add_sta_cmd_size(mvm), &cmd); if (ret) IWL_ERR(mvm, "Failed to send ADD_STA command (%d)\n", ret); @@ -4152,7 +4150,8 @@ void iwl_mvm_sta_modify_disable_tx(struct iwl_mvm *mvm, int ret; if (mvm->mld_api_is_used) { - iwl_mvm_mld_sta_modify_disable_tx(mvm, mvmsta, disable); + if (!iwl_mvm_has_no_host_disable_tx(mvm)) + iwl_mvm_mld_sta_modify_disable_tx(mvm, mvmsta, disable); return; } @@ -4169,7 +4168,8 @@ void iwl_mvm_sta_modify_disable_tx_ap(struct iwl_mvm *mvm, struct iwl_mvm_sta *mvm_sta = iwl_mvm_sta_from_mac80211(sta); if (mvm->mld_api_is_used) { - iwl_mvm_mld_sta_modify_disable_tx_ap(mvm, sta, disable); + if (!iwl_mvm_has_no_host_disable_tx(mvm)) + iwl_mvm_mld_sta_modify_disable_tx_ap(mvm, sta, disable); return; } @@ -4224,7 +4224,9 @@ void iwl_mvm_modify_all_sta_disable_tx(struct iwl_mvm *mvm, int i; if (mvm->mld_api_is_used) { - iwl_mvm_mld_modify_all_sta_disable_tx(mvm, mvmvif, disable); + if (!iwl_mvm_has_no_host_disable_tx(mvm)) + iwl_mvm_mld_modify_all_sta_disable_tx(mvm, mvmvif, + disable); return; } diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c index 26a0953603ab4f..2c9b98c8184b52 100644 --- a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c +++ b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c @@ -1121,9 +1121,8 @@ static const struct iwl_dev_info iwl_dev_info_table[] = { /* * Read rf id and cdb info from prph register and store it */ -static int get_crf_id(struct iwl_trans *iwl_trans) +static void get_crf_id(struct iwl_trans *iwl_trans) { - int ret = 0; u32 sd_reg_ver_addr; u32 val = 0; @@ -1150,8 +1149,6 @@ static int get_crf_id(struct iwl_trans *iwl_trans) IWL_INFO(iwl_trans, "Detected crf-id 0x%x, cnv-id 0x%x wfpm id 0x%x\n", iwl_trans->hw_crf_id, iwl_trans->hw_cnv_id, iwl_trans->hw_wfpm_id); - - return ret; } /* diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/rx.c b/drivers/net/wireless/intel/iwlwifi/pcie/rx.c index bc6a9f861711fd..0f405ded1a7aa0 100644 --- a/drivers/net/wireless/intel/iwlwifi/pcie/rx.c +++ b/drivers/net/wireless/intel/iwlwifi/pcie/rx.c @@ -1351,8 +1351,7 @@ static void iwl_pcie_rx_handle_rb(struct iwl_trans *trans, if (len < sizeof(*pkt) || offset > max_len) break; - trace_iwlwifi_dev_rx(trans->dev, trans, pkt, len); - trace_iwlwifi_dev_rx_data(trans->dev, trans, pkt, len); + maybe_trace_iwlwifi_dev_rx(trans, pkt, len); /* Reclaim a command buffer only if this packet is a response * to a (driver-originated) command. diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/trans.c b/drivers/net/wireless/intel/iwlwifi/pcie/trans.c index 92253260f56832..fc64e1e7f5eeb9 100644 --- a/drivers/net/wireless/intel/iwlwifi/pcie/trans.c +++ b/drivers/net/wireless/intel/iwlwifi/pcie/trans.c @@ -2107,18 +2107,29 @@ static void iwl_trans_pcie_removal_wk(struct work_struct *wk) container_of(wk, struct iwl_trans_pcie_removal, work); struct pci_dev *pdev = removal->pdev; static char *prop[] = {"EVENT=INACCESSIBLE", NULL}; - struct pci_bus *bus = pdev->bus; + struct pci_bus *bus; + + pci_lock_rescan_remove(); + + bus = pdev->bus; + /* in this case, something else already removed the device */ + if (!bus) + goto out; dev_err(&pdev->dev, "Device gone - attempting removal\n"); + kobject_uevent_env(&pdev->dev.kobj, KOBJ_CHANGE, prop); - pci_lock_rescan_remove(); - pci_dev_put(pdev); + pci_stop_and_remove_bus_device(pdev); - if (removal->rescan && bus) { + pci_dev_put(pdev); + + if (removal->rescan) { if (bus->parent) bus = bus->parent; pci_rescan_bus(bus); } + +out: pci_unlock_rescan_remove(); kfree(removal); @@ -2133,6 +2144,7 @@ void iwl_trans_pcie_remove(struct iwl_trans *trans, bool rescan) return; IWL_ERR(trans, "Device gone - scheduling removal!\n"); + iwl_pcie_dump_csr(trans); /* * get a module reference to avoid doing this @@ -2365,32 +2377,6 @@ static int iwl_trans_pcie_read_config32(struct iwl_trans *trans, u32 ofs, ofs, val); } -static void iwl_trans_pcie_block_txq_ptrs(struct iwl_trans *trans, bool block) -{ - int i; - - for (i = 0; i < trans->trans_cfg->base_params->num_of_queues; i++) { - struct iwl_txq *txq = trans->txqs.txq[i]; - - if (i == trans->txqs.cmd.q_id) - continue; - - spin_lock_bh(&txq->lock); - - if (!block && !(WARN_ON_ONCE(!txq->block))) { - txq->block--; - if (!txq->block) { - iwl_write32(trans, HBUS_TARG_WRPTR, - txq->write_ptr | (i << 8)); - } - } else if (block) { - txq->block++; - } - - spin_unlock_bh(&txq->lock); - } -} - #define IWL_FLUSH_WAIT_MS 2000 static int iwl_trans_pcie_rxq_dma_data(struct iwl_trans *trans, int queue, @@ -3572,7 +3558,6 @@ static const struct iwl_trans_ops trans_ops_pcie = { .wait_tx_queues_empty = iwl_trans_pcie_wait_txqs_empty, .freeze_txq_timer = iwl_trans_txq_freeze_timer, - .block_txq_ptrs = iwl_trans_pcie_block_txq_ptrs, #ifdef CONFIG_IWLWIFI_DEBUGFS .debugfs_cleanup = iwl_trans_pcie_debugfs_cleanup, #endif diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/tx-gen2.c b/drivers/net/wireless/intel/iwlwifi/pcie/tx-gen2.c index c72a84d8bb4f73..aabbef114bc23b 100644 --- a/drivers/net/wireless/intel/iwlwifi/pcie/tx-gen2.c +++ b/drivers/net/wireless/intel/iwlwifi/pcie/tx-gen2.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* * Copyright (C) 2017 Intel Deutschland GmbH - * Copyright (C) 2018-2020 Intel Corporation + * Copyright (C) 2018-2020, 2023 Intel Corporation */ #include #include @@ -42,6 +42,9 @@ int iwl_pcie_gen2_enqueue_hcmd(struct iwl_trans *trans, struct iwl_tfh_tfd *tfd; unsigned long flags; + if (WARN_ON(cmd->flags & CMD_BLOCK_TXQS)) + return -EINVAL; + copy_size = sizeof(struct iwl_cmd_header_wide); cmd_size = sizeof(struct iwl_cmd_header_wide); diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/tx.c b/drivers/net/wireless/intel/iwlwifi/pcie/tx.c index 2f39b639c43f99..6c2b37e56c7861 100644 --- a/drivers/net/wireless/intel/iwlwifi/pcie/tx.c +++ b/drivers/net/wireless/intel/iwlwifi/pcie/tx.c @@ -873,6 +873,33 @@ void iwl_trans_pcie_txq_disable(struct iwl_trans *trans, int txq_id, /*************** HOST COMMAND QUEUE FUNCTIONS *****/ +static void iwl_trans_pcie_block_txq_ptrs(struct iwl_trans *trans, bool block) +{ + int i; + + for (i = 0; i < trans->trans_cfg->base_params->num_of_queues; i++) { + struct iwl_txq *txq = trans->txqs.txq[i]; + + if (i == trans->txqs.cmd.q_id) + continue; + + /* we skip the command queue (obviously) so it's OK to nest */ + spin_lock_nested(&txq->lock, 1); + + if (!block && !(WARN_ON_ONCE(!txq->block))) { + txq->block--; + if (!txq->block) { + iwl_write32(trans, HBUS_TARG_WRPTR, + txq->write_ptr | (i << 8)); + } + } else if (block) { + txq->block++; + } + + spin_unlock(&txq->lock); + } +} + /* * iwl_pcie_enqueue_hcmd - enqueue a uCode command * @priv: device private data point @@ -1137,6 +1164,9 @@ int iwl_pcie_enqueue_hcmd(struct iwl_trans *trans, goto out; } + if (cmd->flags & CMD_BLOCK_TXQS) + iwl_trans_pcie_block_txq_ptrs(trans, true); + /* Increment and update queue's write index */ txq->write_ptr = iwl_txq_inc_wrap(trans, txq->write_ptr); iwl_pcie_txq_inc_wr_ptr(trans, txq); @@ -1202,8 +1232,8 @@ void iwl_pcie_hcmd_complete(struct iwl_trans *trans, meta->source->_rx_page_order = trans_pcie->rx_page_order; } - if (meta->flags & CMD_WANT_ASYNC_CALLBACK) - iwl_op_mode_async_cb(trans->op_mode, cmd); + if (meta->flags & CMD_BLOCK_TXQS) + iwl_trans_pcie_block_txq_ptrs(trans, false); iwl_pcie_cmdq_reclaim(trans, txq_id, index); diff --git a/drivers/net/wireless/marvell/libertas/Kconfig b/drivers/net/wireless/marvell/libertas/Kconfig index 56156a021be39f..36b234bc5be805 100644 --- a/drivers/net/wireless/marvell/libertas/Kconfig +++ b/drivers/net/wireless/marvell/libertas/Kconfig @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only config LIBERTAS tristate "Marvell 8xxx Libertas WLAN driver support" - depends on USB || SDIO || SPI + depends on USB || MMC || SPI depends on CFG80211 select LIB80211 select FW_LOADER diff --git a/drivers/net/wireless/marvell/mwifiex/cfg80211.c b/drivers/net/wireless/marvell/mwifiex/cfg80211.c index 7a15ea8072e6f9..3604abcbcff932 100644 --- a/drivers/net/wireless/marvell/mwifiex/cfg80211.c +++ b/drivers/net/wireless/marvell/mwifiex/cfg80211.c @@ -2047,6 +2047,8 @@ static int mwifiex_cfg80211_start_ap(struct wiphy *wiphy, mwifiex_set_sys_config_invalid_data(bss_cfg); + memcpy(bss_cfg->mac_addr, priv->curr_addr, ETH_ALEN); + if (params->beacon_interval) bss_cfg->beacon_period = params->beacon_interval; if (params->dtim_period) diff --git a/drivers/net/wireless/marvell/mwifiex/fw.h b/drivers/net/wireless/marvell/mwifiex/fw.h index 8e6db904e5b2d8..62f3c9a52a1d5c 100644 --- a/drivers/net/wireless/marvell/mwifiex/fw.h +++ b/drivers/net/wireless/marvell/mwifiex/fw.h @@ -165,6 +165,7 @@ enum MWIFIEX_802_11_PRIVACY_FILTER { #define TLV_TYPE_STA_MAC_ADDR (PROPRIETARY_TLV_BASE_ID + 32) #define TLV_TYPE_BSSID (PROPRIETARY_TLV_BASE_ID + 35) #define TLV_TYPE_CHANNELBANDLIST (PROPRIETARY_TLV_BASE_ID + 42) +#define TLV_TYPE_UAP_MAC_ADDRESS (PROPRIETARY_TLV_BASE_ID + 43) #define TLV_TYPE_UAP_BEACON_PERIOD (PROPRIETARY_TLV_BASE_ID + 44) #define TLV_TYPE_UAP_DTIM_PERIOD (PROPRIETARY_TLV_BASE_ID + 45) #define TLV_TYPE_UAP_BCAST_SSID (PROPRIETARY_TLV_BASE_ID + 48) diff --git a/drivers/net/wireless/marvell/mwifiex/ioctl.h b/drivers/net/wireless/marvell/mwifiex/ioctl.h index 091e7ca7937620..e8825f302de8a3 100644 --- a/drivers/net/wireless/marvell/mwifiex/ioctl.h +++ b/drivers/net/wireless/marvell/mwifiex/ioctl.h @@ -107,6 +107,7 @@ struct mwifiex_uap_bss_param { u8 qos_info; u8 power_constraint; struct mwifiex_types_wmm_info wmm_info; + u8 mac_addr[ETH_ALEN]; }; enum { diff --git a/drivers/net/wireless/marvell/mwifiex/sdio.c b/drivers/net/wireless/marvell/mwifiex/sdio.c index 6462a0ffe698c3..ef3e68d1059c5b 100644 --- a/drivers/net/wireless/marvell/mwifiex/sdio.c +++ b/drivers/net/wireless/marvell/mwifiex/sdio.c @@ -331,6 +331,7 @@ static const struct mwifiex_sdio_device mwifiex_sdio_sd8786 = { .can_dump_fw = false, .can_auto_tdls = false, .can_ext_scan = false, + .fw_ready_extra_delay = false, }; static const struct mwifiex_sdio_device mwifiex_sdio_sd8787 = { @@ -346,6 +347,7 @@ static const struct mwifiex_sdio_device mwifiex_sdio_sd8787 = { .can_dump_fw = false, .can_auto_tdls = false, .can_ext_scan = true, + .fw_ready_extra_delay = false, }; static const struct mwifiex_sdio_device mwifiex_sdio_sd8797 = { @@ -361,6 +363,7 @@ static const struct mwifiex_sdio_device mwifiex_sdio_sd8797 = { .can_dump_fw = false, .can_auto_tdls = false, .can_ext_scan = true, + .fw_ready_extra_delay = false, }; static const struct mwifiex_sdio_device mwifiex_sdio_sd8897 = { @@ -376,6 +379,7 @@ static const struct mwifiex_sdio_device mwifiex_sdio_sd8897 = { .can_dump_fw = true, .can_auto_tdls = false, .can_ext_scan = true, + .fw_ready_extra_delay = false, }; static const struct mwifiex_sdio_device mwifiex_sdio_sd8977 = { @@ -392,6 +396,7 @@ static const struct mwifiex_sdio_device mwifiex_sdio_sd8977 = { .fw_dump_enh = true, .can_auto_tdls = false, .can_ext_scan = true, + .fw_ready_extra_delay = false, }; static const struct mwifiex_sdio_device mwifiex_sdio_sd8978 = { @@ -408,6 +413,7 @@ static const struct mwifiex_sdio_device mwifiex_sdio_sd8978 = { .fw_dump_enh = true, .can_auto_tdls = false, .can_ext_scan = true, + .fw_ready_extra_delay = true, }; static const struct mwifiex_sdio_device mwifiex_sdio_sd8997 = { @@ -425,6 +431,7 @@ static const struct mwifiex_sdio_device mwifiex_sdio_sd8997 = { .fw_dump_enh = true, .can_auto_tdls = false, .can_ext_scan = true, + .fw_ready_extra_delay = false, }; static const struct mwifiex_sdio_device mwifiex_sdio_sd8887 = { @@ -440,6 +447,7 @@ static const struct mwifiex_sdio_device mwifiex_sdio_sd8887 = { .can_dump_fw = false, .can_auto_tdls = true, .can_ext_scan = true, + .fw_ready_extra_delay = false, }; static const struct mwifiex_sdio_device mwifiex_sdio_sd8987 = { @@ -456,6 +464,7 @@ static const struct mwifiex_sdio_device mwifiex_sdio_sd8987 = { .fw_dump_enh = true, .can_auto_tdls = true, .can_ext_scan = true, + .fw_ready_extra_delay = false, }; static const struct mwifiex_sdio_device mwifiex_sdio_sd8801 = { @@ -471,6 +480,7 @@ static const struct mwifiex_sdio_device mwifiex_sdio_sd8801 = { .can_dump_fw = false, .can_auto_tdls = false, .can_ext_scan = true, + .fw_ready_extra_delay = false, }; static struct memory_type_mapping generic_mem_type_map[] = { @@ -563,6 +573,7 @@ mwifiex_sdio_probe(struct sdio_func *func, const struct sdio_device_id *id) card->fw_dump_enh = data->fw_dump_enh; card->can_auto_tdls = data->can_auto_tdls; card->can_ext_scan = data->can_ext_scan; + card->fw_ready_extra_delay = data->fw_ready_extra_delay; INIT_WORK(&card->work, mwifiex_sdio_work); } @@ -766,6 +777,7 @@ mwifiex_sdio_read_fw_status(struct mwifiex_adapter *adapter, u16 *dat) static int mwifiex_check_fw_status(struct mwifiex_adapter *adapter, u32 poll_num) { + struct sdio_mmc_card *card = adapter->card; int ret = 0; u16 firmware_stat; u32 tries; @@ -783,6 +795,13 @@ static int mwifiex_check_fw_status(struct mwifiex_adapter *adapter, ret = -1; } + if (card->fw_ready_extra_delay && + firmware_stat == FIRMWARE_READY_SDIO) + /* firmware might pretend to be ready, when it's not. + * Wait a little bit more as a workaround. + */ + msleep(100); + return ret; } diff --git a/drivers/net/wireless/marvell/mwifiex/sdio.h b/drivers/net/wireless/marvell/mwifiex/sdio.h index b86a9263a6a812..cb63ad55d675f2 100644 --- a/drivers/net/wireless/marvell/mwifiex/sdio.h +++ b/drivers/net/wireless/marvell/mwifiex/sdio.h @@ -255,6 +255,7 @@ struct sdio_mmc_card { bool fw_dump_enh; bool can_auto_tdls; bool can_ext_scan; + bool fw_ready_extra_delay; struct mwifiex_sdio_mpa_tx mpa_tx; struct mwifiex_sdio_mpa_rx mpa_rx; @@ -278,6 +279,7 @@ struct mwifiex_sdio_device { bool fw_dump_enh; bool can_auto_tdls; bool can_ext_scan; + bool fw_ready_extra_delay; }; /* diff --git a/drivers/net/wireless/marvell/mwifiex/uap_cmd.c b/drivers/net/wireless/marvell/mwifiex/uap_cmd.c index e78a201cd1507d..491e366119096e 100644 --- a/drivers/net/wireless/marvell/mwifiex/uap_cmd.c +++ b/drivers/net/wireless/marvell/mwifiex/uap_cmd.c @@ -468,6 +468,7 @@ void mwifiex_config_uap_11d(struct mwifiex_private *priv, static int mwifiex_uap_bss_param_prepare(u8 *tlv, void *cmd_buf, u16 *param_size) { + struct host_cmd_tlv_mac_addr *mac_tlv; struct host_cmd_tlv_dtim_period *dtim_period; struct host_cmd_tlv_beacon_period *beacon_period; struct host_cmd_tlv_ssid *ssid; @@ -487,6 +488,13 @@ mwifiex_uap_bss_param_prepare(u8 *tlv, void *cmd_buf, u16 *param_size) int i; u16 cmd_size = *param_size; + mac_tlv = (struct host_cmd_tlv_mac_addr *)tlv; + mac_tlv->header.type = cpu_to_le16(TLV_TYPE_UAP_MAC_ADDRESS); + mac_tlv->header.len = cpu_to_le16(ETH_ALEN); + memcpy(mac_tlv->mac_addr, bss_cfg->mac_addr, ETH_ALEN); + cmd_size += sizeof(struct host_cmd_tlv_mac_addr); + tlv += sizeof(struct host_cmd_tlv_mac_addr); + if (bss_cfg->ssid.ssid_len) { ssid = (struct host_cmd_tlv_ssid *)tlv; ssid->header.type = cpu_to_le16(TLV_TYPE_UAP_SSID); diff --git a/drivers/net/wireless/mediatek/mt76/dma.c b/drivers/net/wireless/mediatek/mt76/dma.c index 68ad915203aa54..00230f106294be 100644 --- a/drivers/net/wireless/mediatek/mt76/dma.c +++ b/drivers/net/wireless/mediatek/mt76/dma.c @@ -9,11 +9,11 @@ #if IS_ENABLED(CONFIG_NET_MEDIATEK_SOC_WED) -#define Q_READ(_dev, _q, _field) ({ \ +#define Q_READ(_q, _field) ({ \ u32 _offset = offsetof(struct mt76_queue_regs, _field); \ u32 _val; \ if ((_q)->flags & MT_QFLAG_WED) \ - _val = mtk_wed_device_reg_read(&(_dev)->mmio.wed, \ + _val = mtk_wed_device_reg_read((_q)->wed, \ ((_q)->wed_regs + \ _offset)); \ else \ @@ -21,10 +21,10 @@ _val; \ }) -#define Q_WRITE(_dev, _q, _field, _val) do { \ +#define Q_WRITE(_q, _field, _val) do { \ u32 _offset = offsetof(struct mt76_queue_regs, _field); \ if ((_q)->flags & MT_QFLAG_WED) \ - mtk_wed_device_reg_write(&(_dev)->mmio.wed, \ + mtk_wed_device_reg_write((_q)->wed, \ ((_q)->wed_regs + _offset), \ _val); \ else \ @@ -33,8 +33,8 @@ #else -#define Q_READ(_dev, _q, _field) readl(&(_q)->regs->_field) -#define Q_WRITE(_dev, _q, _field, _val) writel(_val, &(_q)->regs->_field) +#define Q_READ(_q, _field) readl(&(_q)->regs->_field) +#define Q_WRITE(_q, _field, _val) writel(_val, &(_q)->regs->_field) #endif @@ -188,41 +188,67 @@ EXPORT_SYMBOL_GPL(mt76_free_pending_rxwi); static void mt76_dma_sync_idx(struct mt76_dev *dev, struct mt76_queue *q) { - Q_WRITE(dev, q, desc_base, q->desc_dma); - Q_WRITE(dev, q, ring_size, q->ndesc); - q->head = Q_READ(dev, q, dma_idx); + Q_WRITE(q, desc_base, q->desc_dma); + if (q->flags & MT_QFLAG_WED_RRO_EN) + Q_WRITE(q, ring_size, MT_DMA_RRO_EN | q->ndesc); + else + Q_WRITE(q, ring_size, q->ndesc); + q->head = Q_READ(q, dma_idx); q->tail = q->head; } static void -mt76_dma_queue_reset(struct mt76_dev *dev, struct mt76_queue *q) +__mt76_dma_queue_reset(struct mt76_dev *dev, struct mt76_queue *q, + bool reset_idx) { - int i; - if (!q || !q->ndesc) return; - /* clear descriptors */ - for (i = 0; i < q->ndesc; i++) - q->desc[i].ctrl = cpu_to_le32(MT_DMA_CTL_DMA_DONE); + if (!mt76_queue_is_wed_rro_ind(q)) { + int i; + + /* clear descriptors */ + for (i = 0; i < q->ndesc; i++) + q->desc[i].ctrl = cpu_to_le32(MT_DMA_CTL_DMA_DONE); + } - Q_WRITE(dev, q, cpu_idx, 0); - Q_WRITE(dev, q, dma_idx, 0); + if (reset_idx) { + Q_WRITE(q, cpu_idx, 0); + Q_WRITE(q, dma_idx, 0); + } mt76_dma_sync_idx(dev, q); } +static void +mt76_dma_queue_reset(struct mt76_dev *dev, struct mt76_queue *q) +{ + __mt76_dma_queue_reset(dev, q, true); +} + static int mt76_dma_add_rx_buf(struct mt76_dev *dev, struct mt76_queue *q, struct mt76_queue_buf *buf, void *data) { - struct mt76_desc *desc = &q->desc[q->head]; struct mt76_queue_entry *entry = &q->entry[q->head]; struct mt76_txwi_cache *txwi = NULL; - u32 buf1 = 0, ctrl; + struct mt76_desc *desc; int idx = q->head; + u32 buf1 = 0, ctrl; int rx_token; + if (mt76_queue_is_wed_rro_ind(q)) { + struct mt76_wed_rro_desc *rro_desc; + + rro_desc = (struct mt76_wed_rro_desc *)q->desc; + data = &rro_desc[q->head]; + goto done; + } + + desc = &q->desc[q->head]; ctrl = FIELD_PREP(MT_DMA_CTL_SD_LEN0, buf[0].len); +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT + buf1 = FIELD_PREP(MT_DMA_CTL_SDP0_H, buf->addr >> 32); +#endif if (mt76_queue_is_wed_rx(q)) { txwi = mt76_get_rxwi(dev); @@ -244,6 +270,7 @@ mt76_dma_add_rx_buf(struct mt76_dev *dev, struct mt76_queue *q, WRITE_ONCE(desc->ctrl, cpu_to_le32(ctrl)); WRITE_ONCE(desc->info, 0); +done: entry->dma_addr[0] = buf->addr; entry->dma_len[0] = buf->len; entry->txwi = txwi; @@ -288,11 +315,18 @@ mt76_dma_add_buf(struct mt76_dev *dev, struct mt76_queue *q, entry->dma_len[0] = buf[0].len; ctrl = FIELD_PREP(MT_DMA_CTL_SD_LEN0, buf[0].len); +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT + info |= FIELD_PREP(MT_DMA_CTL_SDP0_H, buf[0].addr >> 32); +#endif if (i < nbufs - 1) { entry->dma_addr[1] = buf[1].addr; entry->dma_len[1] = buf[1].len; buf1 = buf[1].addr; ctrl |= FIELD_PREP(MT_DMA_CTL_SD_LEN1, buf[1].len); +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT + info |= FIELD_PREP(MT_DMA_CTL_SDP1_H, + buf[1].addr >> 32); +#endif if (buf[1].skip_unmap) entry->skip_buf1 = true; } @@ -343,7 +377,7 @@ static void mt76_dma_kick_queue(struct mt76_dev *dev, struct mt76_queue *q) { wmb(); - Q_WRITE(dev, q, cpu_idx, q->head); + Q_WRITE(q, cpu_idx, q->head); } static void @@ -359,7 +393,7 @@ mt76_dma_tx_cleanup(struct mt76_dev *dev, struct mt76_queue *q, bool flush) if (flush) last = -1; else - last = Q_READ(dev, q, dma_idx); + last = Q_READ(q, dma_idx); while (q->queued > 0 && q->tail != last) { mt76_dma_tx_cleanup_idx(dev, q, q->tail, &entry); @@ -371,7 +405,7 @@ mt76_dma_tx_cleanup(struct mt76_dev *dev, struct mt76_queue *q, bool flush) } if (!flush && q->tail == last) - last = Q_READ(dev, q, dma_idx); + last = Q_READ(q, dma_idx); } spin_unlock_bh(&q->cleanup_lock); @@ -392,19 +426,26 @@ mt76_dma_get_buf(struct mt76_dev *dev, struct mt76_queue *q, int idx, { struct mt76_queue_entry *e = &q->entry[idx]; struct mt76_desc *desc = &q->desc[idx]; - void *buf; + u32 ctrl, desc_info, buf1; + void *buf = e->buf; + + if (mt76_queue_is_wed_rro_ind(q)) + goto done; + ctrl = le32_to_cpu(READ_ONCE(desc->ctrl)); if (len) { - u32 ctrl = le32_to_cpu(READ_ONCE(desc->ctrl)); *len = FIELD_GET(MT_DMA_CTL_SD_LEN0, ctrl); *more = !(ctrl & MT_DMA_CTL_LAST_SEC0); } + desc_info = le32_to_cpu(desc->info); if (info) - *info = le32_to_cpu(desc->info); + *info = desc_info; + + buf1 = le32_to_cpu(desc->buf1); + mt76_dma_should_drop_buf(drop, ctrl, buf1, desc_info); if (mt76_queue_is_wed_rx(q)) { - u32 buf1 = le32_to_cpu(desc->buf1); u32 token = FIELD_GET(MT_DMA_CTL_TOKEN, buf1); struct mt76_txwi_cache *t = mt76_rx_token_release(dev, token); @@ -420,23 +461,16 @@ mt76_dma_get_buf(struct mt76_dev *dev, struct mt76_queue *q, int idx, t->ptr = NULL; mt76_put_rxwi(dev, t); - - if (drop) { - u32 ctrl = le32_to_cpu(READ_ONCE(desc->ctrl)); - - *drop = !!(ctrl & (MT_DMA_CTL_TO_HOST_A | - MT_DMA_CTL_DROP)); - + if (drop) *drop |= !!(buf1 & MT_DMA_CTL_WO_DROP); - } } else { - buf = e->buf; - e->buf = NULL; dma_sync_single_for_cpu(dev->dma_dev, e->dma_addr[0], SKB_WITH_OVERHEAD(q->buf_size), page_pool_get_dma_dir(q->page_pool)); } +done: + e->buf = NULL; return buf; } @@ -450,11 +484,16 @@ mt76_dma_dequeue(struct mt76_dev *dev, struct mt76_queue *q, bool flush, if (!q->queued) return NULL; - if (flush) - q->desc[idx].ctrl |= cpu_to_le32(MT_DMA_CTL_DMA_DONE); - else if (!(q->desc[idx].ctrl & cpu_to_le32(MT_DMA_CTL_DMA_DONE))) + if (mt76_queue_is_wed_rro_data(q)) return NULL; + if (!mt76_queue_is_wed_rro_ind(q)) { + if (flush) + q->desc[idx].ctrl |= cpu_to_le32(MT_DMA_CTL_DMA_DONE); + else if (!(q->desc[idx].ctrl & cpu_to_le32(MT_DMA_CTL_DMA_DONE))) + return NULL; + } + q->tail = (q->tail + 1) % q->ndesc; q->queued--; @@ -606,11 +645,14 @@ mt76_dma_rx_fill(struct mt76_dev *dev, struct mt76_queue *q, spin_lock_bh(&q->lock); while (q->queued < q->ndesc - 1) { + struct mt76_queue_buf qbuf = {}; enum dma_data_direction dir; - struct mt76_queue_buf qbuf; dma_addr_t addr; int offset; - void *buf; + void *buf = NULL; + + if (mt76_queue_is_wed_rro_ind(q)) + goto done; buf = mt76_get_page_pool_buf(q, &offset, q->buf_size); if (!buf) @@ -621,6 +663,7 @@ mt76_dma_rx_fill(struct mt76_dev *dev, struct mt76_queue *q, dma_sync_single_for_device(dev->dma_dev, addr, len, dir); qbuf.addr = addr + q->buf_offset; +done: qbuf.len = len - q->buf_offset; qbuf.skip_unmap = false; if (mt76_dma_add_rx_buf(dev, q, &qbuf, buf) < 0) { @@ -630,7 +673,7 @@ mt76_dma_rx_fill(struct mt76_dev *dev, struct mt76_queue *q, frames++; } - if (frames) + if (frames || mt76_queue_is_wed_rx(q)) mt76_dma_kick_queue(dev, q); spin_unlock_bh(&q->lock); @@ -641,15 +684,14 @@ mt76_dma_rx_fill(struct mt76_dev *dev, struct mt76_queue *q, int mt76_dma_wed_setup(struct mt76_dev *dev, struct mt76_queue *q, bool reset) { #ifdef CONFIG_NET_MEDIATEK_SOC_WED - struct mtk_wed_device *wed = &dev->mmio.wed; - int ret, type, ring; - u8 flags; + int ret = 0, type, ring; + u16 flags; if (!q || !q->ndesc) return -EINVAL; flags = q->flags; - if (!mtk_wed_device_active(wed)) + if (!q->wed || !mtk_wed_device_active(q->wed)) q->flags &= ~MT_QFLAG_WED; if (!(q->flags & MT_QFLAG_WED)) @@ -660,29 +702,52 @@ int mt76_dma_wed_setup(struct mt76_dev *dev, struct mt76_queue *q, bool reset) switch (type) { case MT76_WED_Q_TX: - ret = mtk_wed_device_tx_ring_setup(wed, ring, q->regs, reset); + ret = mtk_wed_device_tx_ring_setup(q->wed, ring, q->regs, + reset); if (!ret) - q->wed_regs = wed->tx_ring[ring].reg_base; + q->wed_regs = q->wed->tx_ring[ring].reg_base; break; case MT76_WED_Q_TXFREE: /* WED txfree queue needs ring to be initialized before setup */ q->flags = 0; mt76_dma_queue_reset(dev, q); mt76_dma_rx_fill(dev, q, false); - q->flags = flags; - ret = mtk_wed_device_txfree_ring_setup(wed, q->regs); + ret = mtk_wed_device_txfree_ring_setup(q->wed, q->regs); if (!ret) - q->wed_regs = wed->txfree_ring.reg_base; + q->wed_regs = q->wed->txfree_ring.reg_base; break; case MT76_WED_Q_RX: - ret = mtk_wed_device_rx_ring_setup(wed, ring, q->regs, reset); + ret = mtk_wed_device_rx_ring_setup(q->wed, ring, q->regs, + reset); if (!ret) - q->wed_regs = wed->rx_ring[ring].reg_base; + q->wed_regs = q->wed->rx_ring[ring].reg_base; + break; + case MT76_WED_RRO_Q_DATA: + q->flags &= ~MT_QFLAG_WED; + __mt76_dma_queue_reset(dev, q, false); + mtk_wed_device_rro_rx_ring_setup(q->wed, ring, q->regs); + q->head = q->ndesc - 1; + q->queued = q->head; + break; + case MT76_WED_RRO_Q_MSDU_PG: + q->flags &= ~MT_QFLAG_WED; + __mt76_dma_queue_reset(dev, q, false); + mtk_wed_device_msdu_pg_rx_ring_setup(q->wed, ring, q->regs); + q->head = q->ndesc - 1; + q->queued = q->head; + break; + case MT76_WED_RRO_Q_IND: + q->flags &= ~MT_QFLAG_WED; + mt76_dma_queue_reset(dev, q); + mt76_dma_rx_fill(dev, q, false); + mtk_wed_device_ind_rx_ring_setup(q->wed, q->regs); break; default: ret = -EINVAL; + break; } + q->flags = flags; return ret; #else @@ -706,11 +771,26 @@ mt76_dma_alloc_queue(struct mt76_dev *dev, struct mt76_queue *q, q->buf_size = bufsize; q->hw_idx = idx; - size = q->ndesc * sizeof(struct mt76_desc); - q->desc = dmam_alloc_coherent(dev->dma_dev, size, &q->desc_dma, GFP_KERNEL); + size = mt76_queue_is_wed_rro_ind(q) ? sizeof(struct mt76_wed_rro_desc) + : sizeof(struct mt76_desc); + q->desc = dmam_alloc_coherent(dev->dma_dev, q->ndesc * size, + &q->desc_dma, GFP_KERNEL); if (!q->desc) return -ENOMEM; + if (mt76_queue_is_wed_rro_ind(q)) { + struct mt76_wed_rro_desc *rro_desc; + int i; + + rro_desc = (struct mt76_wed_rro_desc *)q->desc; + for (i = 0; i < q->ndesc; i++) { + struct mt76_wed_rro_ind *cmd; + + cmd = (struct mt76_wed_rro_ind *)&rro_desc[i]; + cmd->magic_cnt = MT_DMA_WED_IND_CMD_CNT - 1; + } + } + size = q->ndesc * sizeof(*q->entry); q->entry = devm_kzalloc(dev->dev, size, GFP_KERNEL); if (!q->entry) @@ -724,8 +804,13 @@ mt76_dma_alloc_queue(struct mt76_dev *dev, struct mt76_queue *q, if (ret) return ret; - if (q->flags != MT_WED_Q_TXFREE) - mt76_dma_queue_reset(dev, q); + if (mtk_wed_device_active(&dev->mmio.wed)) { + if ((mtk_wed_get_rx_capa(&dev->mmio.wed) && mt76_queue_is_wed_rro(q)) || + mt76_queue_is_wed_tx_free(q)) + return 0; + } + + mt76_dma_queue_reset(dev, q); return 0; } @@ -747,7 +832,8 @@ mt76_dma_rx_cleanup(struct mt76_dev *dev, struct mt76_queue *q) if (!buf) break; - mt76_put_page_pool_buf(buf, false); + if (!mt76_queue_is_wed_rro(q)) + mt76_put_page_pool_buf(buf, false); } while (1); spin_lock_bh(&q->lock); @@ -763,22 +849,31 @@ static void mt76_dma_rx_reset(struct mt76_dev *dev, enum mt76_rxq_id qid) { struct mt76_queue *q = &dev->q_rx[qid]; - int i; if (!q->ndesc) return; - for (i = 0; i < q->ndesc; i++) - q->desc[i].ctrl = cpu_to_le32(MT_DMA_CTL_DMA_DONE); + if (!mt76_queue_is_wed_rro_ind(q)) { + int i; + + for (i = 0; i < q->ndesc; i++) + q->desc[i].ctrl = cpu_to_le32(MT_DMA_CTL_DMA_DONE); + } mt76_dma_rx_cleanup(dev, q); /* reset WED rx queues */ mt76_dma_wed_setup(dev, q, true); - if (q->flags != MT_WED_Q_TXFREE) { - mt76_dma_sync_idx(dev, q); - mt76_dma_rx_fill(dev, q, false); - } + + if (mt76_queue_is_wed_tx_free(q)) + return; + + if (mtk_wed_device_active(&dev->mmio.wed) && + mt76_queue_is_wed_rro(q)) + return; + + mt76_dma_sync_idx(dev, q); + mt76_dma_rx_fill(dev, q, false); } static void @@ -819,8 +914,8 @@ mt76_dma_rx_process(struct mt76_dev *dev, struct mt76_queue *q, int budget) bool more; if (IS_ENABLED(CONFIG_NET_MEDIATEK_SOC_WED) && - q->flags == MT_WED_Q_TXFREE) { - dma_idx = Q_READ(dev, q, dma_idx); + mt76_queue_is_wed_tx_free(q)) { + dma_idx = Q_READ(q, dma_idx); check_ddone = true; } @@ -830,7 +925,7 @@ mt76_dma_rx_process(struct mt76_dev *dev, struct mt76_queue *q, int budget) if (check_ddone) { if (q->tail == dma_idx) - dma_idx = Q_READ(dev, q, dma_idx); + dma_idx = Q_READ(q, dma_idx); if (q->tail == dma_idx) break; @@ -959,6 +1054,20 @@ void mt76_dma_attach(struct mt76_dev *dev) } EXPORT_SYMBOL_GPL(mt76_dma_attach); +void mt76_dma_wed_reset(struct mt76_dev *dev) +{ + struct mt76_mmio *mmio = &dev->mmio; + + if (!test_bit(MT76_STATE_WED_RESET, &dev->phy.state)) + return; + + complete(&mmio->wed_reset); + + if (!wait_for_completion_timeout(&mmio->wed_reset_complete, 3 * HZ)) + dev_err(dev->dev, "wed reset complete timeout\n"); +} +EXPORT_SYMBOL_GPL(mt76_dma_wed_reset); + void mt76_dma_cleanup(struct mt76_dev *dev) { int i; @@ -983,16 +1092,23 @@ void mt76_dma_cleanup(struct mt76_dev *dev) mt76_for_each_q_rx(dev, i) { struct mt76_queue *q = &dev->q_rx[i]; + if (mtk_wed_device_active(&dev->mmio.wed) && + mt76_queue_is_wed_rro(q)) + continue; + netif_napi_del(&dev->napi[i]); mt76_dma_rx_cleanup(dev, q); page_pool_destroy(q->page_pool); } - mt76_free_pending_txwi(dev); - mt76_free_pending_rxwi(dev); - if (mtk_wed_device_active(&dev->mmio.wed)) mtk_wed_device_detach(&dev->mmio.wed); + + if (mtk_wed_device_active(&dev->mmio.wed_hif2)) + mtk_wed_device_detach(&dev->mmio.wed_hif2); + + mt76_free_pending_txwi(dev); + mt76_free_pending_rxwi(dev); } EXPORT_SYMBOL_GPL(mt76_dma_cleanup); diff --git a/drivers/net/wireless/mediatek/mt76/dma.h b/drivers/net/wireless/mediatek/mt76/dma.h index 1b090d78cd055b..c479cc6388eff4 100644 --- a/drivers/net/wireless/mediatek/mt76/dma.h +++ b/drivers/net/wireless/mediatek/mt76/dma.h @@ -19,12 +19,23 @@ #define MT_DMA_CTL_TO_HOST_A BIT(12) #define MT_DMA_CTL_DROP BIT(14) #define MT_DMA_CTL_TOKEN GENMASK(31, 16) +#define MT_DMA_CTL_SDP1_H GENMASK(19, 16) +#define MT_DMA_CTL_SDP0_H GENMASK(3, 0) #define MT_DMA_CTL_WO_DROP BIT(8) #define MT_DMA_PPE_CPU_REASON GENMASK(15, 11) #define MT_DMA_PPE_ENTRY GENMASK(30, 16) +#define MT_DMA_INFO_DMA_FRAG BIT(9) #define MT_DMA_INFO_PPE_VLD BIT(31) +#define MT_DMA_CTL_PN_CHK_FAIL BIT(13) +#define MT_DMA_CTL_VER_MASK BIT(7) + +#define MT_DMA_RRO_EN BIT(13) + +#define MT_DMA_WED_IND_CMD_CNT 8 +#define MT_DMA_WED_IND_REASON GENMASK(15, 12) + #define MT_DMA_HDR_LEN 4 #define MT_RX_INFO_LEN 4 #define MT_FCE_INFO_LEN 4 @@ -37,6 +48,11 @@ struct mt76_desc { __le32 info; } __packed __aligned(4); +struct mt76_wed_rro_desc { + __le32 buf0; + __le32 buf1; +} __packed __aligned(4); + enum mt76_qsel { MT_QSEL_MGMT, MT_QSEL_HCCA, @@ -54,9 +70,47 @@ enum mt76_mcu_evt_type { EVT_EVENT_DFS_DETECT_RSP, }; +enum mt76_dma_wed_ind_reason { + MT_DMA_WED_IND_REASON_NORMAL, + MT_DMA_WED_IND_REASON_REPEAT, + MT_DMA_WED_IND_REASON_OLDPKT, +}; + int mt76_dma_rx_poll(struct napi_struct *napi, int budget); void mt76_dma_attach(struct mt76_dev *dev); void mt76_dma_cleanup(struct mt76_dev *dev); int mt76_dma_wed_setup(struct mt76_dev *dev, struct mt76_queue *q, bool reset); +void mt76_dma_wed_reset(struct mt76_dev *dev); + +static inline void +mt76_dma_reset_tx_queue(struct mt76_dev *dev, struct mt76_queue *q) +{ + dev->queue_ops->reset_q(dev, q); + if (mtk_wed_device_active(&dev->mmio.wed)) + mt76_dma_wed_setup(dev, q, true); +} + +static inline void +mt76_dma_should_drop_buf(bool *drop, u32 ctrl, u32 buf1, u32 info) +{ + if (!drop) + return; + + *drop = !!(ctrl & (MT_DMA_CTL_TO_HOST_A | MT_DMA_CTL_DROP)); + if (!(ctrl & MT_DMA_CTL_VER_MASK)) + return; + + switch (FIELD_GET(MT_DMA_WED_IND_REASON, buf1)) { + case MT_DMA_WED_IND_REASON_REPEAT: + *drop = true; + break; + case MT_DMA_WED_IND_REASON_OLDPKT: + *drop = !(info & MT_DMA_INFO_DMA_FRAG); + break; + default: + *drop = !!(ctrl & MT_DMA_CTL_PN_CHK_FAIL); + break; + } +} #endif diff --git a/drivers/net/wireless/mediatek/mt76/eeprom.c b/drivers/net/wireless/mediatek/mt76/eeprom.c index 7725dd6763ef26..0bc66cc19acd1e 100644 --- a/drivers/net/wireless/mediatek/mt76/eeprom.c +++ b/drivers/net/wireless/mediatek/mt76/eeprom.c @@ -28,7 +28,7 @@ static int mt76_get_of_eeprom_data(struct mt76_dev *dev, void *eep, int len) return 0; } -static int mt76_get_of_epprom_from_mtd(struct mt76_dev *dev, void *eep, int offset, int len) +int mt76_get_of_data_from_mtd(struct mt76_dev *dev, void *eep, int offset, int len) { #ifdef CONFIG_MTD struct device_node *np = dev->dev->of_node; @@ -67,7 +67,7 @@ static int mt76_get_of_epprom_from_mtd(struct mt76_dev *dev, void *eep, int offs goto out_put_node; } - offset = be32_to_cpup(list); + offset += be32_to_cpup(list); ret = mtd_read(mtd, offset, len, &retlen, eep); put_mtd_device(mtd); if (mtd_is_bitflip(ret)) @@ -105,8 +105,10 @@ static int mt76_get_of_epprom_from_mtd(struct mt76_dev *dev, void *eep, int offs return -ENOENT; #endif } +EXPORT_SYMBOL_GPL(mt76_get_of_data_from_mtd); -static int mt76_get_of_epprom_from_nvmem(struct mt76_dev *dev, void *eep, int len) +int mt76_get_of_data_from_nvmem(struct mt76_dev *dev, void *eep, + const char *cell_name, int len) { struct device_node *np = dev->dev->of_node; struct nvmem_cell *cell; @@ -114,7 +116,7 @@ static int mt76_get_of_epprom_from_nvmem(struct mt76_dev *dev, void *eep, int le size_t retlen; int ret = 0; - cell = of_nvmem_cell_get(np, "eeprom"); + cell = of_nvmem_cell_get(np, cell_name); if (IS_ERR(cell)) return PTR_ERR(cell); @@ -136,8 +138,9 @@ static int mt76_get_of_epprom_from_nvmem(struct mt76_dev *dev, void *eep, int le return ret; } +EXPORT_SYMBOL_GPL(mt76_get_of_data_from_nvmem); -int mt76_get_of_eeprom(struct mt76_dev *dev, void *eep, int offset, int len) +static int mt76_get_of_eeprom(struct mt76_dev *dev, void *eep, int len) { struct device_node *np = dev->dev->of_node; int ret; @@ -149,13 +152,12 @@ int mt76_get_of_eeprom(struct mt76_dev *dev, void *eep, int offset, int len) if (!ret) return 0; - ret = mt76_get_of_epprom_from_mtd(dev, eep, offset, len); + ret = mt76_get_of_data_from_mtd(dev, eep, 0, len); if (!ret) return 0; - return mt76_get_of_epprom_from_nvmem(dev, eep, len); + return mt76_get_of_data_from_nvmem(dev, eep, "eeprom", len); } -EXPORT_SYMBOL_GPL(mt76_get_of_eeprom); void mt76_eeprom_override(struct mt76_phy *phy) @@ -379,7 +381,7 @@ s8 mt76_get_rate_power_limits(struct mt76_phy *phy, if (!np) return target_power; - txs_delta = mt76_get_txs_delta(np, hweight8(phy->antenna_mask)); + txs_delta = mt76_get_txs_delta(np, hweight16(phy->chainmask)); val = mt76_get_of_array(np, "rates-cck", &len, ARRAY_SIZE(dest->cck)); mt76_apply_array_limit(dest->cck, ARRAY_SIZE(dest->cck), val, @@ -412,6 +414,6 @@ mt76_eeprom_init(struct mt76_dev *dev, int len) if (!dev->eeprom.data) return -ENOMEM; - return !mt76_get_of_eeprom(dev, dev->eeprom.data, 0, len); + return !mt76_get_of_eeprom(dev, dev->eeprom.data, len); } EXPORT_SYMBOL_GPL(mt76_eeprom_init); diff --git a/drivers/net/wireless/mediatek/mt76/mac80211.c b/drivers/net/wireless/mediatek/mt76/mac80211.c index 51a767121b0da7..8a3a90d1bfac58 100644 --- a/drivers/net/wireless/mediatek/mt76/mac80211.c +++ b/drivers/net/wireless/mediatek/mt76/mac80211.c @@ -197,10 +197,33 @@ static int mt76_led_init(struct mt76_phy *phy) { struct mt76_dev *dev = phy->dev; struct ieee80211_hw *hw = phy->hw; + struct device_node *np = dev->dev->of_node; if (!phy->leds.cdev.brightness_set && !phy->leds.cdev.blink_set) return 0; + np = of_get_child_by_name(np, "led"); + if (np) { + if (!of_device_is_available(np)) { + of_node_put(np); + dev_info(dev->dev, + "led registration was explicitly disabled by dts\n"); + return 0; + } + + if (phy == &dev->phy) { + int led_pin; + + if (!of_property_read_u32(np, "led-sources", &led_pin)) + phy->leds.pin = led_pin; + + phy->leds.al = + of_property_read_bool(np, "led-active-low"); + } + + of_node_put(np); + } + snprintf(phy->leds.name, sizeof(phy->leds.name), "mt76-%s", wiphy_name(hw->wiphy)); @@ -211,20 +234,8 @@ static int mt76_led_init(struct mt76_phy *phy) mt76_tpt_blink, ARRAY_SIZE(mt76_tpt_blink)); - if (phy == &dev->phy) { - struct device_node *np = dev->dev->of_node; - - np = of_get_child_by_name(np, "led"); - if (np) { - int led_pin; - - if (!of_property_read_u32(np, "led-sources", &led_pin)) - phy->leds.pin = led_pin; - phy->leds.al = of_property_read_bool(np, - "led-active-low"); - of_node_put(np); - } - } + dev_info(dev->dev, + "registering led '%s'\n", phy->leds.name); return led_classdev_register(dev->dev, &phy->leds.cdev); } @@ -1537,7 +1548,7 @@ int mt76_get_txpower(struct ieee80211_hw *hw, struct ieee80211_vif *vif, int *dbm) { struct mt76_phy *phy = hw->priv; - int n_chains = hweight8(phy->antenna_mask); + int n_chains = hweight16(phy->chainmask); int delta = mt76_tx_power_nss_delta(n_chains); *dbm = DIV_ROUND_UP(phy->txpower_cur + delta, 2); @@ -1725,7 +1736,7 @@ EXPORT_SYMBOL_GPL(mt76_get_antenna); struct mt76_queue * mt76_init_queue(struct mt76_dev *dev, int qid, int idx, int n_desc, - int ring_base, u32 flags) + int ring_base, void *wed, u32 flags) { struct mt76_queue *hwq; int err; @@ -1735,6 +1746,7 @@ mt76_init_queue(struct mt76_dev *dev, int qid, int idx, int n_desc, return ERR_PTR(-ENOMEM); hwq->flags = flags; + hwq->wed = wed; err = dev->queue_ops->alloc(dev, hwq, idx, n_desc, 0, ring_base); if (err < 0) @@ -1842,3 +1854,19 @@ enum mt76_dfs_state mt76_phy_dfs_state(struct mt76_phy *phy) return MT_DFS_STATE_ACTIVE; } EXPORT_SYMBOL_GPL(mt76_phy_dfs_state); + +#ifdef CONFIG_NET_MEDIATEK_SOC_WED +int mt76_net_setup_tc(struct ieee80211_hw *hw, struct ieee80211_vif *vif, + struct net_device *netdev, enum tc_setup_type type, + void *type_data) +{ + struct mt76_phy *phy = hw->priv; + struct mtk_wed_device *wed = &phy->dev->mmio.wed; + + if (!mtk_wed_device_active(wed)) + return -EOPNOTSUPP; + + return mtk_wed_device_setup_tc(wed, netdev, type, type_data); +} +EXPORT_SYMBOL_GPL(mt76_net_setup_tc); +#endif /* CONFIG_NET_MEDIATEK_SOC_WED */ diff --git a/drivers/net/wireless/mediatek/mt76/mmio.c b/drivers/net/wireless/mediatek/mt76/mmio.c index 86e3d2ac4d0db2..c3e0e23e0161ac 100644 --- a/drivers/net/wireless/mediatek/mt76/mmio.c +++ b/drivers/net/wireless/mediatek/mt76/mmio.c @@ -4,6 +4,7 @@ */ #include "mt76.h" +#include "dma.h" #include "trace.h" static u32 mt76_mmio_rr(struct mt76_dev *dev, u32 offset) @@ -84,6 +85,113 @@ void mt76_set_irq_mask(struct mt76_dev *dev, u32 addr, } EXPORT_SYMBOL_GPL(mt76_set_irq_mask); +#ifdef CONFIG_NET_MEDIATEK_SOC_WED +void mt76_mmio_wed_release_rx_buf(struct mtk_wed_device *wed) +{ + struct mt76_dev *dev = container_of(wed, struct mt76_dev, mmio.wed); + int i; + + for (i = 0; i < dev->rx_token_size; i++) { + struct mt76_txwi_cache *t; + + t = mt76_rx_token_release(dev, i); + if (!t || !t->ptr) + continue; + + mt76_put_page_pool_buf(t->ptr, false); + t->ptr = NULL; + + mt76_put_rxwi(dev, t); + } + + mt76_free_pending_rxwi(dev); +} +EXPORT_SYMBOL_GPL(mt76_mmio_wed_release_rx_buf); + +u32 mt76_mmio_wed_init_rx_buf(struct mtk_wed_device *wed, int size) +{ + struct mt76_dev *dev = container_of(wed, struct mt76_dev, mmio.wed); + struct mtk_wed_bm_desc *desc = wed->rx_buf_ring.desc; + struct mt76_queue *q = &dev->q_rx[MT_RXQ_MAIN]; + int i, len = SKB_WITH_OVERHEAD(q->buf_size); + struct mt76_txwi_cache *t = NULL; + + for (i = 0; i < size; i++) { + enum dma_data_direction dir; + dma_addr_t addr; + u32 offset; + int token; + void *buf; + + t = mt76_get_rxwi(dev); + if (!t) + goto unmap; + + buf = mt76_get_page_pool_buf(q, &offset, q->buf_size); + if (!buf) + goto unmap; + + addr = page_pool_get_dma_addr(virt_to_head_page(buf)) + offset; + dir = page_pool_get_dma_dir(q->page_pool); + dma_sync_single_for_device(dev->dma_dev, addr, len, dir); + + desc->buf0 = cpu_to_le32(addr); + token = mt76_rx_token_consume(dev, buf, t, addr); + if (token < 0) { + mt76_put_page_pool_buf(buf, false); + goto unmap; + } + + token = FIELD_PREP(MT_DMA_CTL_TOKEN, token); +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT + token |= FIELD_PREP(MT_DMA_CTL_SDP0_H, addr >> 32); +#endif + desc->token |= cpu_to_le32(token); + desc++; + } + + return 0; + +unmap: + if (t) + mt76_put_rxwi(dev, t); + mt76_mmio_wed_release_rx_buf(wed); + + return -ENOMEM; +} +EXPORT_SYMBOL_GPL(mt76_mmio_wed_init_rx_buf); + +int mt76_mmio_wed_offload_enable(struct mtk_wed_device *wed) +{ + struct mt76_dev *dev = container_of(wed, struct mt76_dev, mmio.wed); + + spin_lock_bh(&dev->token_lock); + dev->token_size = wed->wlan.token_start; + spin_unlock_bh(&dev->token_lock); + + return !wait_event_timeout(dev->tx_wait, !dev->wed_token_count, HZ); +} +EXPORT_SYMBOL_GPL(mt76_mmio_wed_offload_enable); + +void mt76_mmio_wed_offload_disable(struct mtk_wed_device *wed) +{ + struct mt76_dev *dev = container_of(wed, struct mt76_dev, mmio.wed); + + spin_lock_bh(&dev->token_lock); + dev->token_size = dev->drv->token_size; + spin_unlock_bh(&dev->token_lock); +} +EXPORT_SYMBOL_GPL(mt76_mmio_wed_offload_disable); + +void mt76_mmio_wed_reset_complete(struct mtk_wed_device *wed) +{ + struct mt76_dev *dev = container_of(wed, struct mt76_dev, mmio.wed); + + complete(&dev->mmio.wed_reset_complete); +} +EXPORT_SYMBOL_GPL(mt76_mmio_wed_reset_complete); +#endif /*CONFIG_NET_MEDIATEK_SOC_WED */ + void mt76_mmio_init(struct mt76_dev *dev, void __iomem *regs) { static const struct mt76_bus_ops mt76_mmio_ops = { diff --git a/drivers/net/wireless/mediatek/mt76/mt76.h b/drivers/net/wireless/mediatek/mt76/mt76.h index ea828ba0b83acf..b20c34d5a0f7cc 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76.h +++ b/drivers/net/wireless/mediatek/mt76/mt76.h @@ -29,15 +29,22 @@ #define MT76_TOKEN_FREE_THR 64 #define MT_QFLAG_WED_RING GENMASK(1, 0) -#define MT_QFLAG_WED_TYPE GENMASK(3, 2) -#define MT_QFLAG_WED BIT(4) +#define MT_QFLAG_WED_TYPE GENMASK(4, 2) +#define MT_QFLAG_WED BIT(5) +#define MT_QFLAG_WED_RRO BIT(6) +#define MT_QFLAG_WED_RRO_EN BIT(7) #define __MT_WED_Q(_type, _n) (MT_QFLAG_WED | \ FIELD_PREP(MT_QFLAG_WED_TYPE, _type) | \ FIELD_PREP(MT_QFLAG_WED_RING, _n)) +#define __MT_WED_RRO_Q(_type, _n) (MT_QFLAG_WED_RRO | __MT_WED_Q(_type, _n)) + #define MT_WED_Q_TX(_n) __MT_WED_Q(MT76_WED_Q_TX, _n) #define MT_WED_Q_RX(_n) __MT_WED_Q(MT76_WED_Q_RX, _n) #define MT_WED_Q_TXFREE __MT_WED_Q(MT76_WED_Q_TXFREE, 0) +#define MT_WED_RRO_Q_DATA(_n) __MT_WED_RRO_Q(MT76_WED_RRO_Q_DATA, _n) +#define MT_WED_RRO_Q_MSDU_PG(_n) __MT_WED_RRO_Q(MT76_WED_RRO_Q_MSDU_PG, _n) +#define MT_WED_RRO_Q_IND __MT_WED_RRO_Q(MT76_WED_RRO_Q_IND, 0) struct mt76_dev; struct mt76_phy; @@ -59,6 +66,9 @@ enum mt76_wed_type { MT76_WED_Q_TX, MT76_WED_Q_TXFREE, MT76_WED_Q_RX, + MT76_WED_RRO_Q_DATA, + MT76_WED_RRO_Q_MSDU_PG, + MT76_WED_RRO_Q_IND, }; struct mt76_bus_ops { @@ -107,6 +117,16 @@ enum mt76_rxq_id { MT_RXQ_MAIN_WA, MT_RXQ_BAND2, MT_RXQ_BAND2_WA, + MT_RXQ_RRO_BAND0, + MT_RXQ_RRO_BAND1, + MT_RXQ_RRO_BAND2, + MT_RXQ_MSDU_PAGE_BAND0, + MT_RXQ_MSDU_PAGE_BAND1, + MT_RXQ_MSDU_PAGE_BAND2, + MT_RXQ_TXFREE_BAND0, + MT_RXQ_TXFREE_BAND1, + MT_RXQ_TXFREE_BAND2, + MT_RXQ_RRO_IND, __MT_RXQ_MAX }; @@ -163,7 +183,7 @@ struct mt76_queue_entry { struct urb *urb; int buf_sz; }; - u32 dma_addr[2]; + dma_addr_t dma_addr[2]; u16 dma_len[2]; u16 wcid; bool skip_buf0:1; @@ -184,6 +204,7 @@ struct mt76_queue { spinlock_t lock; spinlock_t cleanup_lock; struct mt76_queue_entry *entry; + struct mt76_rro_desc *rro_desc; struct mt76_desc *desc; u16 first; @@ -197,8 +218,9 @@ struct mt76_queue { u8 buf_offset; u8 hw_idx; - u8 flags; + u16 flags; + struct mtk_wed_device *wed; u32 wed_regs; dma_addr_t desc_dma; @@ -353,6 +375,17 @@ struct mt76_txq { bool aggr; }; +struct mt76_wed_rro_ind { + u32 se_id : 12; + u32 rsv : 4; + u32 start_sn : 12; + u32 ind_reason : 4; + u32 ind_cnt : 13; + u32 win_sz : 3; + u32 rsv2 : 13; + u32 magic_cnt : 3; +}; + struct mt76_txwi_cache { struct list_head list; dma_addr_t dma_addr; @@ -371,6 +404,7 @@ struct mt76_rx_tid { spinlock_t lock; struct delayed_work reorder_work; + u16 id; u16 head; u16 size; u16 nframes; @@ -575,8 +609,7 @@ struct mt76_sdio { struct mt76_worker txrx_worker; struct mt76_worker status_worker; struct mt76_worker net_worker; - - struct work_struct stat_work; + struct mt76_worker stat_worker; u8 *xmit_buf; u32 xmit_buf_sz; @@ -603,6 +636,7 @@ struct mt76_mmio { u32 irqmask; struct mtk_wed_device wed; + struct mtk_wed_device wed_hif2; struct completion wed_reset; struct completion wed_reset_complete; }; @@ -1047,6 +1081,12 @@ bool ____mt76_poll_msec(struct mt76_dev *dev, u32 offset, u32 mask, u32 val, void mt76_mmio_init(struct mt76_dev *dev, void __iomem *regs); void mt76_pci_disable_aspm(struct pci_dev *pdev); +#ifdef CONFIG_NET_MEDIATEK_SOC_WED +int mt76_net_setup_tc(struct ieee80211_hw *hw, struct ieee80211_vif *vif, + struct net_device *netdev, enum tc_setup_type type, + void *type_data); +#endif /*CONFIG_NET_MEDIATEK_SOC_WED */ + static inline u16 mt76_chip(struct mt76_dev *dev) { return dev->rev >> 16; @@ -1057,6 +1097,14 @@ static inline u16 mt76_rev(struct mt76_dev *dev) return dev->rev & 0xffff; } +#ifdef CONFIG_NET_MEDIATEK_SOC_WED +u32 mt76_mmio_wed_init_rx_buf(struct mtk_wed_device *wed, int size); +void mt76_mmio_wed_release_rx_buf(struct mtk_wed_device *wed); +int mt76_mmio_wed_offload_enable(struct mtk_wed_device *wed); +void mt76_mmio_wed_offload_disable(struct mtk_wed_device *wed); +void mt76_mmio_wed_reset_complete(struct mtk_wed_device *wed); +#endif /*CONFIG_NET_MEDIATEK_SOC_WED */ + #define mt76xx_chip(dev) mt76_chip(&((dev)->mt76)) #define mt76xx_rev(dev) mt76_rev(&((dev)->mt76)) @@ -1102,19 +1150,22 @@ void mt76_seq_puts_array(struct seq_file *file, const char *str, int mt76_eeprom_init(struct mt76_dev *dev, int len); void mt76_eeprom_override(struct mt76_phy *phy); -int mt76_get_of_eeprom(struct mt76_dev *dev, void *data, int offset, int len); +int mt76_get_of_data_from_mtd(struct mt76_dev *dev, void *eep, int offset, int len); +int mt76_get_of_data_from_nvmem(struct mt76_dev *dev, void *eep, + const char *cell_name, int len); struct mt76_queue * mt76_init_queue(struct mt76_dev *dev, int qid, int idx, int n_desc, - int ring_base, u32 flags); + int ring_base, void *wed, u32 flags); u16 mt76_calculate_default_rate(struct mt76_phy *phy, struct ieee80211_vif *vif, int rateidx); static inline int mt76_init_tx_queue(struct mt76_phy *phy, int qid, int idx, - int n_desc, int ring_base, u32 flags) + int n_desc, int ring_base, void *wed, + u32 flags) { struct mt76_queue *q; - q = mt76_init_queue(phy->dev, qid, idx, n_desc, ring_base, flags); + q = mt76_init_queue(phy->dev, qid, idx, n_desc, ring_base, wed, flags); if (IS_ERR(q)) return PTR_ERR(q); @@ -1128,7 +1179,7 @@ static inline int mt76_init_mcu_queue(struct mt76_dev *dev, int qid, int idx, { struct mt76_queue *q; - q = mt76_init_queue(dev, qid, idx, n_desc, ring_base, 0); + q = mt76_init_queue(dev, qid, idx, n_desc, ring_base, NULL, 0); if (IS_ERR(q)) return PTR_ERR(q); @@ -1547,10 +1598,38 @@ s8 mt76_get_rate_power_limits(struct mt76_phy *phy, struct mt76_power_limits *dest, s8 target_power); -static inline bool mt76_queue_is_wed_rx(struct mt76_queue *q) +static inline bool mt76_queue_is_wed_tx_free(struct mt76_queue *q) { return (q->flags & MT_QFLAG_WED) && - FIELD_GET(MT_QFLAG_WED_TYPE, q->flags) == MT76_WED_Q_RX; + FIELD_GET(MT_QFLAG_WED_TYPE, q->flags) == MT76_WED_Q_TXFREE; +} + +static inline bool mt76_queue_is_wed_rro(struct mt76_queue *q) +{ + return q->flags & MT_QFLAG_WED_RRO; +} + +static inline bool mt76_queue_is_wed_rro_ind(struct mt76_queue *q) +{ + return mt76_queue_is_wed_rro(q) && + FIELD_GET(MT_QFLAG_WED_TYPE, q->flags) == MT76_WED_RRO_Q_IND; +} + +static inline bool mt76_queue_is_wed_rro_data(struct mt76_queue *q) +{ + return mt76_queue_is_wed_rro(q) && + (FIELD_GET(MT_QFLAG_WED_TYPE, q->flags) == MT76_WED_RRO_Q_DATA || + FIELD_GET(MT_QFLAG_WED_TYPE, q->flags) == MT76_WED_RRO_Q_MSDU_PG); +} + +static inline bool mt76_queue_is_wed_rx(struct mt76_queue *q) +{ + if (!(q->flags & MT_QFLAG_WED)) + return false; + + return FIELD_GET(MT_QFLAG_WED_TYPE, q->flags) == MT76_WED_Q_RX || + mt76_queue_is_wed_rro_ind(q) || mt76_queue_is_wed_rro_data(q); + } struct mt76_txwi_cache * diff --git a/drivers/net/wireless/mediatek/mt76/mt7603/dma.c b/drivers/net/wireless/mediatek/mt76/mt7603/dma.c index 03ba11a61c90c1..7a2f5d38562b4f 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7603/dma.c +++ b/drivers/net/wireless/mediatek/mt76/mt7603/dma.c @@ -173,13 +173,14 @@ int mt7603_dma_init(struct mt7603_dev *dev) for (i = 0; i < ARRAY_SIZE(wmm_queue_map); i++) { ret = mt76_init_tx_queue(&dev->mphy, i, wmm_queue_map[i], - MT7603_TX_RING_SIZE, MT_TX_RING_BASE, 0); + MT7603_TX_RING_SIZE, MT_TX_RING_BASE, + NULL, 0); if (ret) return ret; } ret = mt76_init_tx_queue(&dev->mphy, MT_TXQ_PSD, MT_TX_HW_QUEUE_MGMT, - MT7603_PSD_RING_SIZE, MT_TX_RING_BASE, 0); + MT7603_PSD_RING_SIZE, MT_TX_RING_BASE, NULL, 0); if (ret) return ret; @@ -189,12 +190,12 @@ int mt7603_dma_init(struct mt7603_dev *dev) return ret; ret = mt76_init_tx_queue(&dev->mphy, MT_TXQ_BEACON, MT_TX_HW_QUEUE_BCN, - MT_MCU_RING_SIZE, MT_TX_RING_BASE, 0); + MT_MCU_RING_SIZE, MT_TX_RING_BASE, NULL, 0); if (ret) return ret; ret = mt76_init_tx_queue(&dev->mphy, MT_TXQ_CAB, MT_TX_HW_QUEUE_BMC, - MT_MCU_RING_SIZE, MT_TX_RING_BASE, 0); + MT_MCU_RING_SIZE, MT_TX_RING_BASE, NULL, 0); if (ret) return ret; diff --git a/drivers/net/wireless/mediatek/mt76/mt7603/soc.c b/drivers/net/wireless/mediatek/mt76/mt7603/soc.c index ba927033bbe8c3..ec02148a7f1f74 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7603/soc.c +++ b/drivers/net/wireless/mediatek/mt76/mt7603/soc.c @@ -52,15 +52,12 @@ mt76_wmac_probe(struct platform_device *pdev) return ret; } -static int -mt76_wmac_remove(struct platform_device *pdev) +static void mt76_wmac_remove(struct platform_device *pdev) { struct mt76_dev *mdev = platform_get_drvdata(pdev); struct mt7603_dev *dev = container_of(mdev, struct mt7603_dev, mt76); mt7603_unregister_device(dev); - - return 0; } static const struct of_device_id of_wmac_match[] = { @@ -74,7 +71,7 @@ MODULE_FIRMWARE(MT7628_FIRMWARE_E2); struct platform_driver mt76_wmac_driver = { .probe = mt76_wmac_probe, - .remove = mt76_wmac_remove, + .remove_new = mt76_wmac_remove, .driver = { .name = "mt76_wmac", .of_match_table = of_wmac_match, diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/dma.c b/drivers/net/wireless/mediatek/mt76/mt7615/dma.c index 0ce01ccc5dcedf..e7135b2f174286 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7615/dma.c +++ b/drivers/net/wireless/mediatek/mt76/mt7615/dma.c @@ -26,14 +26,14 @@ mt7622_init_tx_queues_multi(struct mt7615_dev *dev) for (i = 0; i < ARRAY_SIZE(wmm_queue_map); i++) { ret = mt76_init_tx_queue(&dev->mphy, i, wmm_queue_map[i], MT7615_TX_RING_SIZE / 2, - MT_TX_RING_BASE, 0); + MT_TX_RING_BASE, NULL, 0); if (ret) return ret; } ret = mt76_init_tx_queue(&dev->mphy, MT_TXQ_PSD, MT7622_TXQ_MGMT, MT7615_TX_MGMT_RING_SIZE, - MT_TX_RING_BASE, 0); + MT_TX_RING_BASE, NULL, 0); if (ret) return ret; @@ -55,7 +55,7 @@ mt7615_init_tx_queues(struct mt7615_dev *dev) return mt7622_init_tx_queues_multi(dev); ret = mt76_connac_init_tx_queues(&dev->mphy, 0, MT7615_TX_RING_SIZE, - MT_TX_RING_BASE, 0); + MT_TX_RING_BASE, NULL, 0); if (ret) return ret; diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7615/mcu.c index 955974a82180fd..ae34d019e58838 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7615/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7615/mcu.c @@ -453,7 +453,7 @@ mt7615_mcu_scan_event(struct mt7615_dev *dev, struct sk_buff *skb) else mphy = &dev->mt76.phy; - phy = (struct mt7615_phy *)mphy->priv; + phy = mphy->priv; spin_lock_bh(&dev->mt76.lock); __skb_queue_tail(&phy->scan_event_list, skb); @@ -481,7 +481,7 @@ mt7615_mcu_roc_event(struct mt7615_dev *dev, struct sk_buff *skb) ieee80211_ready_on_channel(mphy->hw); - phy = (struct mt7615_phy *)mphy->priv; + phy = mphy->priv; phy->roc_grant = true; wake_up(&phy->roc_wait); diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/sdio.c b/drivers/net/wireless/mediatek/mt76/mt7615/sdio.c index fc547a0031eae2..67cedd2555f973 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7615/sdio.c +++ b/drivers/net/wireless/mediatek/mt76/mt7615/sdio.c @@ -204,8 +204,8 @@ static int mt7663s_suspend(struct device *dev) mt76_worker_disable(&mdev->mt76.sdio.txrx_worker); mt76_worker_disable(&mdev->mt76.sdio.status_worker); mt76_worker_disable(&mdev->mt76.sdio.net_worker); + mt76_worker_disable(&mdev->mt76.sdio.stat_worker); - cancel_work_sync(&mdev->mt76.sdio.stat_work); clear_bit(MT76_READING_STATS, &mdev->mphy.state); mt76_tx_status_check(&mdev->mt76, true); diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/soc.c b/drivers/net/wireless/mediatek/mt76/mt7615/soc.c index f13d1b41874268..12e3e4a91d2748 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7615/soc.c +++ b/drivers/net/wireless/mediatek/mt76/mt7615/soc.c @@ -45,13 +45,11 @@ static int mt7622_wmac_probe(struct platform_device *pdev) return mt7615_mmio_probe(&pdev->dev, mem_base, irq, mt7615e_reg_map); } -static int mt7622_wmac_remove(struct platform_device *pdev) +static void mt7622_wmac_remove(struct platform_device *pdev) { struct mt7615_dev *dev = platform_get_drvdata(pdev); mt7615_unregister_device(dev); - - return 0; } static const struct of_device_id mt7622_wmac_of_match[] = { @@ -65,7 +63,7 @@ struct platform_driver mt7622_wmac_driver = { .of_match_table = mt7622_wmac_of_match, }, .probe = mt7622_wmac_probe, - .remove = mt7622_wmac_remove, + .remove_new = mt7622_wmac_remove, }; MODULE_FIRMWARE(MT7622_FIRMWARE_N9); diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac.h b/drivers/net/wireless/mediatek/mt76/mt76_connac.h index 1f29d8cd900cba..fdde3d70b300e3 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76_connac.h +++ b/drivers/net/wireless/mediatek/mt76/mt76_connac.h @@ -222,6 +222,11 @@ static inline bool is_mt7996(struct mt76_dev *dev) return mt76_chip(dev) == 0x7990; } +static inline bool is_mt7992(struct mt76_dev *dev) +{ + return mt76_chip(dev) == 0x7992; +} + static inline bool is_mt7622(struct mt76_dev *dev) { if (!IS_ENABLED(CONFIG_MT7622_WMAC)) @@ -391,7 +396,8 @@ mt76_connac_mutex_release(struct mt76_dev *dev, struct mt76_connac_pm *pm) void mt76_connac_gen_ppe_thresh(u8 *he_ppet, int nss); int mt76_connac_init_tx_queues(struct mt76_phy *phy, int idx, int n_desc, - int ring_base, u32 flags); + int ring_base, void *wed, u32 flags); + void mt76_connac_write_hw_txp(struct mt76_dev *dev, struct mt76_tx_info *tx_info, void *txp_ptr, u32 id); diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac3_mac.h b/drivers/net/wireless/mediatek/mt76/mt76_connac3_mac.h index 2250252b2047e7..83dcd964bfd045 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76_connac3_mac.h +++ b/drivers/net/wireless/mediatek/mt76/mt76_connac3_mac.h @@ -239,11 +239,13 @@ enum tx_mgnt_type { #define MT_TXD6_TX_SRC GENMASK(31, 30) #define MT_TXD6_VTA BIT(28) -#define MT_TXD6_BW GENMASK(25, 22) +#define MT_TXD6_FIXED_BW BIT(25) +#define MT_TXD6_BW GENMASK(24, 22) #define MT_TXD6_TX_RATE GENMASK(21, 16) #define MT_TXD6_TIMESTAMP_OFS_EN BIT(15) #define MT_TXD6_TIMESTAMP_OFS_IDX GENMASK(14, 10) #define MT_TXD6_MSDU_CNT GENMASK(9, 4) +#define MT_TXD6_MSDU_CNT_V2 GENMASK(15, 10) #define MT_TXD6_DIS_MAT BIT(3) #define MT_TXD6_DAS BIT(2) #define MT_TXD6_AMSDU_CAP BIT(1) @@ -259,6 +261,9 @@ enum tx_mgnt_type { #define MT_TXD9_WLAN_IDX GENMASK(23, 8) +#define MT_TXP_BUF_LEN GENMASK(11, 0) +#define MT_TXP_DMA_ADDR_H GENMASK(15, 12) + #define MT_TX_RATE_STBC BIT(14) #define MT_TX_RATE_NSS GENMASK(13, 10) #define MT_TX_RATE_MODE GENMASK(9, 6) diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c b/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c index 93402d2c253894..c7914643e9c058 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c @@ -256,11 +256,12 @@ void mt76_connac_txp_skb_unmap(struct mt76_dev *dev, EXPORT_SYMBOL_GPL(mt76_connac_txp_skb_unmap); int mt76_connac_init_tx_queues(struct mt76_phy *phy, int idx, int n_desc, - int ring_base, u32 flags) + int ring_base, void *wed, u32 flags) { int i, err; - err = mt76_init_tx_queue(phy, 0, idx, n_desc, ring_base, flags); + err = mt76_init_tx_queue(phy, 0, idx, n_desc, ring_base, + wed, flags); if (err < 0) return err; diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c index ae6bf3c968dfb7..96494ba2fdf767 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c @@ -67,7 +67,8 @@ int mt76_connac_mcu_init_download(struct mt76_dev *dev, u32 addr, u32 len, if ((!is_connac_v1(dev) && addr == MCU_PATCH_ADDRESS) || (is_mt7921(dev) && addr == 0x900000) || (is_mt7925(dev) && addr == 0x900000) || - (is_mt7996(dev) && addr == 0x900000)) + (is_mt7996(dev) && addr == 0x900000) || + (is_mt7992(dev) && addr == 0x900000)) cmd = MCU_CMD(PATCH_START_REQ); else cmd = MCU_CMD(TARGET_ADDRESS_LEN_REQ); @@ -1359,7 +1360,7 @@ u8 mt76_connac_get_phy_mode_ext(struct mt76_phy *phy, struct ieee80211_vif *vif, sband = phy->hw->wiphy->bands[band]; eht_cap = ieee80211_get_eht_iftype_cap(sband, vif->type); - if (!eht_cap || !eht_cap->has_eht) + if (!eht_cap || !eht_cap->has_eht || !vif->bss_conf.eht_support) return mode; switch (band) { diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.h b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.h index 0563b1b22f485b..ae6d0179727dfb 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.h +++ b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.h @@ -416,6 +416,14 @@ struct sta_rec_he_6g_capa { u8 rsv[2]; } __packed; +struct sta_rec_pn_info { + __le16 tag; + __le16 len; + u8 pn[6]; + u8 tsc_type; + u8 rsv; +} __packed; + struct sec_key { u8 cipher_id; u8 cipher_len; @@ -768,6 +776,7 @@ struct wtbl_raw { sizeof(struct sta_rec_sec) + \ sizeof(struct sta_rec_ra_fixed) + \ sizeof(struct sta_rec_he_6g_capa) + \ + sizeof(struct sta_rec_pn_info) + \ sizeof(struct tlv) + \ MT76_CONNAC_WTBL_UPDATE_MAX_SIZE) @@ -798,6 +807,7 @@ enum { STA_REC_HE_V2 = 0x19, STA_REC_MLD = 0x20, STA_REC_EHT = 0x22, + STA_REC_PN_INFO = 0x26, STA_REC_HDRT = 0x28, STA_REC_HDR_TRANS = 0x2B, STA_REC_MAX_NUM @@ -1021,7 +1031,9 @@ enum { MCU_UNI_EVENT_RDD_REPORT = 0x11, MCU_UNI_EVENT_ROC = 0x27, MCU_UNI_EVENT_TX_DONE = 0x2d, + MCU_UNI_EVENT_THERMAL = 0x35, MCU_UNI_EVENT_NIC_CAPAB = 0x43, + MCU_UNI_EVENT_WED_RRO = 0x57, MCU_UNI_EVENT_PER_STA_INFO = 0x6d, MCU_UNI_EVENT_ALL_STA_INFO = 0x6e, }; @@ -1088,6 +1100,13 @@ enum mcu_cipher_type { MCU_CIPHER_GCMP_256, MCU_CIPHER_WAPI, MCU_CIPHER_BIP_CMAC_128, + MCU_CIPHER_BIP_CMAC_256, + MCU_CIPHER_BCN_PROT_CMAC_128, + MCU_CIPHER_BCN_PROT_CMAC_256, + MCU_CIPHER_BCN_PROT_GMAC_128, + MCU_CIPHER_BCN_PROT_GMAC_256, + MCU_CIPHER_BIP_GMAC_128, + MCU_CIPHER_BIP_GMAC_256, }; enum { @@ -1240,6 +1259,7 @@ enum { MCU_UNI_CMD_CHANNEL_SWITCH = 0x34, MCU_UNI_CMD_THERMAL = 0x35, MCU_UNI_CMD_VOW = 0x37, + MCU_UNI_CMD_FIXED_RATE_TABLE = 0x40, MCU_UNI_CMD_RRO = 0x57, MCU_UNI_CMD_OFFCH_SCAN_CTRL = 0x58, MCU_UNI_CMD_PER_STA_INFO = 0x6d, @@ -1307,6 +1327,7 @@ enum { UNI_BSS_INFO_RATE = 11, UNI_BSS_INFO_QBSS = 15, UNI_BSS_INFO_SEC = 16, + UNI_BSS_INFO_BCN_PROT = 17, UNI_BSS_INFO_TXCMD = 18, UNI_BSS_INFO_UAPSD = 19, UNI_BSS_INFO_PS = 21, @@ -1325,7 +1346,7 @@ enum { }; enum UNI_ALL_STA_INFO_TAG { - UNI_ALL_STA_TX_RATE, + UNI_ALL_STA_TXRX_RATE, UNI_ALL_STA_TX_STAT, UNI_ALL_STA_TXRX_ADM_STAT, UNI_ALL_STA_TXRX_AIR_TIME, @@ -1768,6 +1789,12 @@ mt76_connac_mcu_get_cipher(int cipher) return MCU_CIPHER_GCMP; case WLAN_CIPHER_SUITE_GCMP_256: return MCU_CIPHER_GCMP_256; + case WLAN_CIPHER_SUITE_BIP_GMAC_128: + return MCU_CIPHER_BIP_GMAC_128; + case WLAN_CIPHER_SUITE_BIP_GMAC_256: + return MCU_CIPHER_BIP_GMAC_256; + case WLAN_CIPHER_SUITE_BIP_CMAC_256: + return MCU_CIPHER_BIP_CMAC_256; case WLAN_CIPHER_SUITE_SMS4: return MCU_CIPHER_WAPI; default: diff --git a/drivers/net/wireless/mediatek/mt76/mt76x02_mmio.c b/drivers/net/wireless/mediatek/mt76/mt76x02_mmio.c index 9b5e3fb7b0dfe4..e5ad635d3c5657 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76x02_mmio.c +++ b/drivers/net/wireless/mediatek/mt76/mt76x02_mmio.c @@ -199,13 +199,14 @@ int mt76x02_dma_init(struct mt76x02_dev *dev) for (i = 0; i < IEEE80211_NUM_ACS; i++) { ret = mt76_init_tx_queue(&dev->mphy, i, mt76_ac_to_hwq(i), MT76x02_TX_RING_SIZE, - MT_TX_RING_BASE, 0); + MT_TX_RING_BASE, NULL, 0); if (ret) return ret; } ret = mt76_init_tx_queue(&dev->mphy, MT_TXQ_PSD, MT_TX_HW_QUEUE_MGMT, - MT76x02_PSD_RING_SIZE, MT_TX_RING_BASE, 0); + MT76x02_PSD_RING_SIZE, MT_TX_RING_BASE, + NULL, 0); if (ret) return ret; diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/dma.c b/drivers/net/wireless/mediatek/mt76/mt7915/dma.c index 59a44d79aaedb1..c91a1c54027fa3 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/dma.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/dma.c @@ -9,18 +9,20 @@ static int mt7915_init_tx_queues(struct mt7915_phy *phy, int idx, int n_desc, int ring_base) { struct mt7915_dev *dev = phy->dev; + struct mtk_wed_device *wed = NULL; - if (mtk_wed_device_active(&phy->dev->mt76.mmio.wed)) { + if (mtk_wed_device_active(&dev->mt76.mmio.wed)) { if (is_mt798x(&dev->mt76)) ring_base += MT_TXQ_ID(0) * MT_RING_SIZE; else ring_base = MT_WED_TX_RING_BASE; idx -= MT_TXQ_ID(0); + wed = &dev->mt76.mmio.wed; } return mt76_connac_init_tx_queues(phy->mt76, idx, n_desc, ring_base, - MT_WED_Q_TX(idx)); + wed, MT_WED_Q_TX(idx)); } static int mt7915_poll_tx(struct napi_struct *napi, int budget) @@ -492,7 +494,8 @@ int mt7915_dma_init(struct mt7915_dev *dev, struct mt7915_phy *phy2) if (mtk_wed_device_active(&mdev->mmio.wed) && is_mt7915(mdev)) { wa_rx_base = MT_WED_RX_RING_BASE; wa_rx_idx = MT7915_RXQ_MCU_WA; - dev->mt76.q_rx[MT_RXQ_MCU_WA].flags = MT_WED_Q_TXFREE; + mdev->q_rx[MT_RXQ_MCU_WA].flags = MT_WED_Q_TXFREE; + mdev->q_rx[MT_RXQ_MCU_WA].wed = &mdev->mmio.wed; } else { wa_rx_base = MT_RXQ_RING_BASE(MT_RXQ_MCU_WA); wa_rx_idx = MT_RXQ_ID(MT_RXQ_MCU_WA); @@ -507,9 +510,10 @@ int mt7915_dma_init(struct mt7915_dev *dev, struct mt7915_phy *phy2) if (!dev->phy.mt76->band_idx) { if (mtk_wed_device_active(&mdev->mmio.wed) && mtk_wed_get_rx_capa(&mdev->mmio.wed)) { - dev->mt76.q_rx[MT_RXQ_MAIN].flags = + mdev->q_rx[MT_RXQ_MAIN].flags = MT_WED_Q_RX(MT7915_RXQ_BAND0); dev->mt76.rx_token_size += MT7915_RX_RING_SIZE; + mdev->q_rx[MT_RXQ_MAIN].wed = &mdev->mmio.wed; } ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_MAIN], @@ -528,6 +532,7 @@ int mt7915_dma_init(struct mt7915_dev *dev, struct mt7915_phy *phy2) if (mtk_wed_device_active(&mdev->mmio.wed)) { mdev->q_rx[MT_RXQ_MAIN_WA].flags = MT_WED_Q_TXFREE; + mdev->q_rx[MT_RXQ_MAIN_WA].wed = &mdev->mmio.wed; if (is_mt7916(mdev)) { wa_rx_base = MT_WED_RX_RING_BASE; wa_rx_idx = MT7915_RXQ_MCU_WA; @@ -544,9 +549,10 @@ int mt7915_dma_init(struct mt7915_dev *dev, struct mt7915_phy *phy2) if (dev->dbdc_support || dev->phy.mt76->band_idx) { if (mtk_wed_device_active(&mdev->mmio.wed) && mtk_wed_get_rx_capa(&mdev->mmio.wed)) { - dev->mt76.q_rx[MT_RXQ_BAND1].flags = + mdev->q_rx[MT_RXQ_BAND1].flags = MT_WED_Q_RX(MT7915_RXQ_BAND1); dev->mt76.rx_token_size += MT7915_RX_RING_SIZE; + mdev->q_rx[MT_RXQ_BAND1].wed = &mdev->mmio.wed; } /* rx data queue for band1 */ @@ -581,28 +587,6 @@ int mt7915_dma_init(struct mt7915_dev *dev, struct mt7915_phy *phy2) return 0; } -static void mt7915_dma_wed_reset(struct mt7915_dev *dev) -{ - struct mt76_dev *mdev = &dev->mt76; - - if (!test_bit(MT76_STATE_WED_RESET, &dev->mphy.state)) - return; - - complete(&mdev->mmio.wed_reset); - - if (!wait_for_completion_timeout(&dev->mt76.mmio.wed_reset_complete, - 3 * HZ)) - dev_err(dev->mt76.dev, "wed reset complete timeout\n"); -} - -static void -mt7915_dma_reset_tx_queue(struct mt7915_dev *dev, struct mt76_queue *q) -{ - mt76_queue_reset(dev, q); - if (mtk_wed_device_active(&dev->mt76.mmio.wed)) - mt76_dma_wed_setup(&dev->mt76, q, true); -} - int mt7915_dma_reset(struct mt7915_dev *dev, bool force) { struct mt76_phy *mphy_ext = dev->mt76.phys[MT_BAND1]; @@ -630,20 +614,20 @@ int mt7915_dma_reset(struct mt7915_dev *dev, bool force) mtk_wed_device_dma_reset(wed); mt7915_dma_disable(dev, force); - mt7915_dma_wed_reset(dev); + mt76_dma_wed_reset(&dev->mt76); /* reset hw queues */ for (i = 0; i < __MT_TXQ_MAX; i++) { - mt7915_dma_reset_tx_queue(dev, dev->mphy.q_tx[i]); + mt76_dma_reset_tx_queue(&dev->mt76, dev->mphy.q_tx[i]); if (mphy_ext) - mt7915_dma_reset_tx_queue(dev, mphy_ext->q_tx[i]); + mt76_dma_reset_tx_queue(&dev->mt76, mphy_ext->q_tx[i]); } for (i = 0; i < __MT_MCUQ_MAX; i++) mt76_queue_reset(dev, dev->mt76.q_mcu[i]); mt76_for_each_q_rx(&dev->mt76, i) { - if (dev->mt76.q_rx[i].flags == MT_WED_Q_TXFREE) + if (mt76_queue_is_wed_tx_free(&dev->mt76.q_rx[i])) continue; mt76_queue_reset(dev, &dev->mt76.q_rx[i]); diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/eeprom.c b/drivers/net/wireless/mediatek/mt76/mt7915/eeprom.c index 76be7308460b06..3bb2643d1b2696 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/eeprom.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/eeprom.c @@ -11,6 +11,7 @@ static int mt7915_eeprom_load_precal(struct mt7915_dev *dev) u8 *eeprom = mdev->eeprom.data; u32 val = eeprom[MT_EE_DO_PRE_CAL]; u32 offs; + int ret; if (!dev->flash_mode) return 0; @@ -25,7 +26,11 @@ static int mt7915_eeprom_load_precal(struct mt7915_dev *dev) offs = is_mt7915(&dev->mt76) ? MT_EE_PRECAL : MT_EE_PRECAL_V2; - return mt76_get_of_eeprom(mdev, dev->cal, offs, val); + ret = mt76_get_of_data_from_mtd(mdev, dev->cal, offs, val); + if (!ret) + return ret; + + return mt76_get_of_data_from_nvmem(mdev, dev->cal, "precal", val); } static int mt7915_check_eeprom(struct mt7915_dev *dev) diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/eeprom.h b/drivers/net/wireless/mediatek/mt76/mt7915/eeprom.h index f3e56817d36e9d..adc26a222823bb 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/eeprom.h +++ b/drivers/net/wireless/mediatek/mt76/mt7915/eeprom.h @@ -144,7 +144,8 @@ static inline bool mt7915_tssi_enabled(struct mt7915_dev *dev, enum nl80211_band band) { u8 *eep = dev->mt76.eeprom.data; - u8 val = eep[MT_EE_WIFI_CONF + 7]; + u8 offs = is_mt7981(&dev->mt76) ? 8 : 7; + u8 val = eep[MT_EE_WIFI_CONF + offs]; if (band == NL80211_BAND_2GHZ) return val & MT_EE_WIFI_CONF7_TSSI0_2G; diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/init.c b/drivers/net/wireless/mediatek/mt76/mt7915/init.c index 81478289f17e89..cea2f6d9050af7 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/init.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/init.c @@ -275,10 +275,11 @@ static void mt7915_led_set_brightness(struct led_classdev *led_cdev, mt7915_led_set_config(led_cdev, 0xff, 0); } -void mt7915_init_txpower(struct mt7915_dev *dev, - struct ieee80211_supported_band *sband) +static void __mt7915_init_txpower(struct mt7915_phy *phy, + struct ieee80211_supported_band *sband) { - int i, n_chains = hweight8(dev->mphy.antenna_mask); + struct mt7915_dev *dev = phy->dev; + int i, n_chains = hweight16(phy->mt76->chainmask); int nss_delta = mt76_tx_power_nss_delta(n_chains); int pwr_delta = mt7915_eeprom_get_power_delta(dev, sband->band); struct mt76_power_limits limits; @@ -296,7 +297,7 @@ void mt7915_init_txpower(struct mt7915_dev *dev, } target_power += pwr_delta; - target_power = mt76_get_rate_power_limits(&dev->mphy, chan, + target_power = mt76_get_rate_power_limits(phy->mt76, chan, &limits, target_power); target_power += nss_delta; @@ -307,6 +308,19 @@ void mt7915_init_txpower(struct mt7915_dev *dev, } } +void mt7915_init_txpower(struct mt7915_phy *phy) +{ + if (!phy) + return; + + if (phy->mt76->cap.has_2ghz) + __mt7915_init_txpower(phy, &phy->mt76->sband_2g.sband); + if (phy->mt76->cap.has_5ghz) + __mt7915_init_txpower(phy, &phy->mt76->sband_5g.sband); + if (phy->mt76->cap.has_6ghz) + __mt7915_init_txpower(phy, &phy->mt76->sband_6g.sband); +} + static void mt7915_regd_notifier(struct wiphy *wiphy, struct regulatory_request *request) @@ -322,9 +336,7 @@ mt7915_regd_notifier(struct wiphy *wiphy, if (dev->mt76.region == NL80211_DFS_UNSET) mt7915_mcu_rdd_background_enable(phy, NULL); - mt7915_init_txpower(dev, &mphy->sband_2g.sband); - mt7915_init_txpower(dev, &mphy->sband_5g.sband); - mt7915_init_txpower(dev, &mphy->sband_6g.sband); + mt7915_init_txpower(phy); mphy->dfs_state = MT_DFS_STATE_UNKNOWN; mt7915_dfs_init_radar_detector(phy); @@ -442,6 +454,7 @@ mt7915_init_wiphy(struct mt7915_phy *phy) mt76_set_stream_caps(phy->mt76, true); mt7915_set_stream_vht_txbf_caps(phy); mt7915_set_stream_he_caps(phy); + mt7915_init_txpower(phy); wiphy->available_antennas_rx = phy->mt76->antenna_mask; wiphy->available_antennas_tx = phy->mt76->antenna_mask; @@ -703,9 +716,6 @@ static void mt7915_init_work(struct work_struct *work) mt7915_mcu_set_eeprom(dev); mt7915_mac_init(dev); - mt7915_init_txpower(dev, &dev->mphy.sband_2g.sband); - mt7915_init_txpower(dev, &dev->mphy.sband_5g.sband); - mt7915_init_txpower(dev, &dev->mphy.sband_6g.sband); mt7915_txbf_init(dev); } diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mac.c b/drivers/net/wireless/mediatek/mt76/mt7915/mac.c index 2222fb9aa103ef..b01edbed969cac 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mac.c @@ -1247,7 +1247,7 @@ mt7915_phy_get_nf(struct mt7915_phy *phy, int idx) void mt7915_update_channel(struct mt76_phy *mphy) { - struct mt7915_phy *phy = (struct mt7915_phy *)mphy->priv; + struct mt7915_phy *phy = mphy->priv; struct mt76_channel_state *state = mphy->chan_state; int nf; @@ -1401,8 +1401,8 @@ mt7915_mac_restart(struct mt7915_dev *dev) goto out; mt7915_mac_init(dev); - mt7915_init_txpower(dev, &dev->mphy.sband_2g.sband); - mt7915_init_txpower(dev, &dev->mphy.sband_5g.sband); + mt7915_init_txpower(&dev->phy); + mt7915_init_txpower(phy2); ret = mt7915_txbf_init(dev); if (test_bit(MT76_STATE_RUNNING, &dev->mphy.state)) { diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/main.c b/drivers/net/wireless/mediatek/mt76/mt7915/main.c index a3fd54cc1911af..df2d4279790d9d 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/main.c @@ -1059,8 +1059,9 @@ mt7915_set_antenna(struct ieee80211_hw *hw, u32 tx_ant, u32 rx_ant) phy->mt76->antenna_mask = tx_ant; - /* handle a variant of mt7916 which has 3T3R but nss2 on 5 GHz band */ - if (is_mt7916(&dev->mt76) && band && hweight8(tx_ant) == max_nss) + /* handle a variant of mt7916/mt7981 which has 3T3R but nss2 on 5 GHz band */ + if ((is_mt7916(&dev->mt76) || is_mt7981(&dev->mt76)) && + band && hweight8(tx_ant) == max_nss) phy->mt76->chainmask = (dev->chainmask >> chainshift) << chainshift; else phy->mt76->chainmask = tx_ant << (chainshift * band); @@ -1653,20 +1654,6 @@ mt7915_net_fill_forward_path(struct ieee80211_hw *hw, return 0; } - -static int -mt7915_net_setup_tc(struct ieee80211_hw *hw, struct ieee80211_vif *vif, - struct net_device *netdev, enum tc_setup_type type, - void *type_data) -{ - struct mt7915_dev *dev = mt7915_hw_dev(hw); - struct mtk_wed_device *wed = &dev->mt76.mmio.wed; - - if (!mtk_wed_device_active(wed)) - return -EOPNOTSUPP; - - return mtk_wed_device_setup_tc(wed, netdev, type, type_data); -} #endif const struct ieee80211_ops mt7915_ops = { @@ -1721,6 +1708,6 @@ const struct ieee80211_ops mt7915_ops = { .set_radar_background = mt7915_set_radar_background, #ifdef CONFIG_NET_MEDIATEK_SOC_WED .net_fill_forward_path = mt7915_net_fill_forward_path, - .net_setup_tc = mt7915_net_setup_tc, + .net_setup_tc = mt76_net_setup_tc, #endif }; diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c index b22f06d4411a82..c67c4f6ca2aae1 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c @@ -269,7 +269,7 @@ mt7915_mcu_rx_thermal_notify(struct mt7915_dev *dev, struct sk_buff *skb) dev->mt76.phys[MT_BAND1]) mphy = dev->mt76.phys[MT_BAND1]; - phy = (struct mt7915_phy *)mphy->priv; + phy = mphy->priv; phy->throttle_state = t->ctrl.duty.duty_cycle; } diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.h b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.h index 1592b5d6751a0f..b41ac4aaced7fb 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.h +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.h @@ -519,7 +519,7 @@ static inline s8 mt7915_get_power_bound(struct mt7915_phy *phy, s8 txpower) { struct mt76_phy *mphy = phy->mt76; - int n_chains = hweight8(mphy->antenna_mask); + int n_chains = hweight16(mphy->chainmask); txpower = mt76_get_sar_power(mphy, mphy->chandef.chan, txpower * 2); txpower -= mt76_tx_power_nss_delta(n_chains); diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mmio.c b/drivers/net/wireless/mediatek/mt76/mt7915/mmio.c index e7d8e03f826f89..aff4f21e843d29 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mmio.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mmio.c @@ -542,105 +542,6 @@ static u32 mt7915_rmw(struct mt76_dev *mdev, u32 offset, u32 mask, u32 val) } #ifdef CONFIG_NET_MEDIATEK_SOC_WED -static int mt7915_mmio_wed_offload_enable(struct mtk_wed_device *wed) -{ - struct mt7915_dev *dev; - - dev = container_of(wed, struct mt7915_dev, mt76.mmio.wed); - - spin_lock_bh(&dev->mt76.token_lock); - dev->mt76.token_size = wed->wlan.token_start; - spin_unlock_bh(&dev->mt76.token_lock); - - return !wait_event_timeout(dev->mt76.tx_wait, - !dev->mt76.wed_token_count, HZ); -} - -static void mt7915_mmio_wed_offload_disable(struct mtk_wed_device *wed) -{ - struct mt7915_dev *dev; - - dev = container_of(wed, struct mt7915_dev, mt76.mmio.wed); - - spin_lock_bh(&dev->mt76.token_lock); - dev->mt76.token_size = MT7915_TOKEN_SIZE; - spin_unlock_bh(&dev->mt76.token_lock); -} - -static void mt7915_mmio_wed_release_rx_buf(struct mtk_wed_device *wed) -{ - struct mt7915_dev *dev; - int i; - - dev = container_of(wed, struct mt7915_dev, mt76.mmio.wed); - for (i = 0; i < dev->mt76.rx_token_size; i++) { - struct mt76_txwi_cache *t; - - t = mt76_rx_token_release(&dev->mt76, i); - if (!t || !t->ptr) - continue; - - mt76_put_page_pool_buf(t->ptr, false); - t->ptr = NULL; - - mt76_put_rxwi(&dev->mt76, t); - } - - mt76_free_pending_rxwi(&dev->mt76); -} - -static u32 mt7915_mmio_wed_init_rx_buf(struct mtk_wed_device *wed, int size) -{ - struct mtk_wed_bm_desc *desc = wed->rx_buf_ring.desc; - struct mt76_txwi_cache *t = NULL; - struct mt7915_dev *dev; - struct mt76_queue *q; - int i, len; - - dev = container_of(wed, struct mt7915_dev, mt76.mmio.wed); - q = &dev->mt76.q_rx[MT_RXQ_MAIN]; - len = SKB_WITH_OVERHEAD(q->buf_size); - - for (i = 0; i < size; i++) { - enum dma_data_direction dir; - dma_addr_t addr; - u32 offset; - int token; - void *buf; - - t = mt76_get_rxwi(&dev->mt76); - if (!t) - goto unmap; - - buf = mt76_get_page_pool_buf(q, &offset, q->buf_size); - if (!buf) - goto unmap; - - addr = page_pool_get_dma_addr(virt_to_head_page(buf)) + offset; - dir = page_pool_get_dma_dir(q->page_pool); - dma_sync_single_for_device(dev->mt76.dma_dev, addr, len, dir); - - desc->buf0 = cpu_to_le32(addr); - token = mt76_rx_token_consume(&dev->mt76, buf, t, addr); - if (token < 0) { - mt76_put_page_pool_buf(buf, false); - goto unmap; - } - - desc->token |= cpu_to_le32(FIELD_PREP(MT_DMA_CTL_TOKEN, - token)); - desc++; - } - - return 0; - -unmap: - if (t) - mt76_put_rxwi(&dev->mt76, t); - mt7915_mmio_wed_release_rx_buf(wed); - return -ENOMEM; -} - static void mt7915_mmio_wed_update_rx_stats(struct mtk_wed_device *wed, struct mtk_wed_wo_rx_stats *stats) { @@ -694,13 +595,6 @@ static int mt7915_mmio_wed_reset(struct mtk_wed_device *wed) return ret; } - -static void mt7915_mmio_wed_reset_complete(struct mtk_wed_device *wed) -{ - struct mt76_dev *dev = container_of(wed, struct mt76_dev, mmio.wed); - - complete(&dev->mmio.wed_reset_complete); -} #endif int mt7915_mmio_wed_init(struct mt7915_dev *dev, void *pdev_ptr, @@ -742,7 +636,7 @@ int mt7915_mmio_wed_init(struct mt7915_dev *dev, void *pdev_ptr, res = platform_get_resource(plat_dev, IORESOURCE_MEM, 0); if (!res) - return -ENOMEM; + return 0; wed->wlan.platform_dev = plat_dev; wed->wlan.bus_type = MTK_WED_BUS_AXI; @@ -778,13 +672,13 @@ int mt7915_mmio_wed_init(struct mt7915_dev *dev, void *pdev_ptr, } wed->wlan.init_buf = mt7915_wed_init_buf; - wed->wlan.offload_enable = mt7915_mmio_wed_offload_enable; - wed->wlan.offload_disable = mt7915_mmio_wed_offload_disable; - wed->wlan.init_rx_buf = mt7915_mmio_wed_init_rx_buf; - wed->wlan.release_rx_buf = mt7915_mmio_wed_release_rx_buf; + wed->wlan.offload_enable = mt76_mmio_wed_offload_enable; + wed->wlan.offload_disable = mt76_mmio_wed_offload_disable; + wed->wlan.init_rx_buf = mt76_mmio_wed_init_rx_buf; + wed->wlan.release_rx_buf = mt76_mmio_wed_release_rx_buf; wed->wlan.update_wo_rx_stats = mt7915_mmio_wed_update_rx_stats; wed->wlan.reset = mt7915_mmio_wed_reset; - wed->wlan.reset_complete = mt7915_mmio_wed_reset_complete; + wed->wlan.reset_complete = mt76_mmio_wed_reset_complete; dev->mt76.rx_token_size = wed->wlan.rx_npkt; diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h b/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h index d317c523b23fab..4727d9c7b11de0 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h @@ -425,8 +425,7 @@ void mt7915_dma_cleanup(struct mt7915_dev *dev); int mt7915_dma_reset(struct mt7915_dev *dev, bool force); int mt7915_dma_start(struct mt7915_dev *dev, bool reset, bool wed_reset); int mt7915_txbf_init(struct mt7915_dev *dev); -void mt7915_init_txpower(struct mt7915_dev *dev, - struct ieee80211_supported_band *sband); +void mt7915_init_txpower(struct mt7915_phy *phy); void mt7915_reset(struct mt7915_dev *dev); int mt7915_run(struct ieee80211_hw *hw); int mt7915_mcu_init(struct mt7915_dev *dev); diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/soc.c b/drivers/net/wireless/mediatek/mt76/mt7915/soc.c index 06e3d9db996c14..8b4809703efcf6 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/soc.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/soc.c @@ -1282,13 +1282,11 @@ static int mt798x_wmac_probe(struct platform_device *pdev) return ret; } -static int mt798x_wmac_remove(struct platform_device *pdev) +static void mt798x_wmac_remove(struct platform_device *pdev) { struct mt7915_dev *dev = platform_get_drvdata(pdev); mt7915_unregister_device(dev); - - return 0; } static const struct of_device_id mt798x_wmac_of_match[] = { @@ -1305,7 +1303,7 @@ struct platform_driver mt798x_wmac_driver = { .of_match_table = mt798x_wmac_of_match, }, .probe = mt798x_wmac_probe, - .remove = mt798x_wmac_remove, + .remove_new = mt798x_wmac_remove, }; MODULE_FIRMWARE(MT7986_FIRMWARE_WA); diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/init.c b/drivers/net/wireless/mediatek/mt76/mt7921/init.c index 7d6a9d74601110..48433c6d5e7d30 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/init.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/init.c @@ -110,24 +110,37 @@ mt7921_regd_channel_update(struct wiphy *wiphy, struct mt792x_dev *dev) } } +void mt7921_regd_update(struct mt792x_dev *dev) +{ + struct mt76_dev *mdev = &dev->mt76; + struct ieee80211_hw *hw = mdev->hw; + struct wiphy *wiphy = hw->wiphy; + + mt7921_mcu_set_clc(dev, mdev->alpha2, dev->country_ie_env); + mt7921_regd_channel_update(wiphy, dev); + mt76_connac_mcu_set_channel_domain(hw->priv); + mt7921_set_tx_sar_pwr(hw, NULL); +} +EXPORT_SYMBOL_GPL(mt7921_regd_update); + static void mt7921_regd_notifier(struct wiphy *wiphy, struct regulatory_request *request) { struct ieee80211_hw *hw = wiphy_to_ieee80211_hw(wiphy); struct mt792x_dev *dev = mt792x_hw_dev(hw); + struct mt76_connac_pm *pm = &dev->pm; memcpy(dev->mt76.alpha2, request->alpha2, sizeof(dev->mt76.alpha2)); dev->mt76.region = request->dfs_region; dev->country_ie_env = request->country_ie_env; + if (pm->suspended) + return; + mt792x_mutex_acquire(dev); - mt7921_mcu_set_clc(dev, request->alpha2, request->country_ie_env); - mt76_connac_mcu_set_channel_domain(hw->priv); - mt7921_set_tx_sar_pwr(hw, NULL); + mt7921_regd_update(dev); mt792x_mutex_release(dev); - - mt7921_regd_channel_update(wiphy, dev); } int mt7921_mac_init(struct mt792x_dev *dev) diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/main.c b/drivers/net/wireless/mediatek/mt76/mt7921/main.c index 510a575a973b8d..0645417e05825f 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/main.c @@ -683,17 +683,45 @@ static void mt7921_bss_info_changed(struct ieee80211_hw *hw, } static void -mt7921_regd_set_6ghz_power_type(struct ieee80211_vif *vif) +mt7921_calc_vif_num(void *priv, u8 *mac, struct ieee80211_vif *vif) +{ + u32 *num = priv; + + if (!priv) + return; + + switch (vif->type) { + case NL80211_IFTYPE_STATION: + case NL80211_IFTYPE_P2P_CLIENT: + case NL80211_IFTYPE_AP: + case NL80211_IFTYPE_P2P_GO: + *num += 1; + break; + default: + break; + } +} + +static void +mt7921_regd_set_6ghz_power_type(struct ieee80211_vif *vif, bool is_add) { struct mt792x_vif *mvif = (struct mt792x_vif *)vif->drv_priv; struct mt792x_phy *phy = mvif->phy; struct mt792x_dev *dev = phy->dev; + u32 valid_vif_num = 0; + + ieee80211_iterate_active_interfaces(mt76_hw(dev), + IEEE80211_IFACE_ITER_RESUME_ALL, + mt7921_calc_vif_num, &valid_vif_num); - if (hweight64(dev->mt76.vif_mask) > 1) { + if (valid_vif_num > 1) { phy->power_type = MT_AP_DEFAULT; goto out; } + if (!is_add) + vif->bss_conf.power_type = IEEE80211_REG_UNSET_AP; + switch (vif->bss_conf.power_type) { case IEEE80211_REG_SP_AP: phy->power_type = MT_AP_SP; @@ -705,6 +733,8 @@ mt7921_regd_set_6ghz_power_type(struct ieee80211_vif *vif) phy->power_type = MT_AP_LPI; break; case IEEE80211_REG_UNSET_AP: + phy->power_type = MT_AP_UNSET; + break; default: phy->power_type = MT_AP_DEFAULT; break; @@ -749,7 +779,7 @@ int mt7921_mac_sta_add(struct mt76_dev *mdev, struct ieee80211_vif *vif, if (ret) return ret; - mt7921_regd_set_6ghz_power_type(vif); + mt7921_regd_set_6ghz_power_type(vif, true); mt76_connac_power_save_sched(&dev->mphy, &dev->pm); @@ -811,6 +841,8 @@ void mt7921_mac_sta_remove(struct mt76_dev *mdev, struct ieee80211_vif *vif, list_del_init(&msta->wcid.poll_list); spin_unlock_bh(&dev->mt76.sta_poll_lock); + mt7921_regd_set_6ghz_power_type(vif, false); + mt76_connac_power_save_sched(&dev->mphy, &dev->pm); } EXPORT_SYMBOL_GPL(mt7921_mac_sta_remove); diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c index 2cc2d2788f8312..f5582477c7e4c7 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c @@ -160,7 +160,7 @@ static void mt7921_mcu_scan_event(struct mt792x_dev *dev, struct sk_buff *skb) { struct mt76_phy *mphy = &dev->mt76.phy; - struct mt792x_phy *phy = (struct mt792x_phy *)mphy->priv; + struct mt792x_phy *phy = mphy->priv; spin_lock_bh(&dev->mt76.lock); __skb_queue_tail(&phy->scan_event_list, skb); @@ -1261,15 +1261,19 @@ int __mt7921_mcu_set_clc(struct mt792x_dev *dev, u8 *alpha2, u8 alpha2[2]; u8 type[2]; u8 env_6g; - u8 rsvd[63]; + u8 mtcl_conf; + u8 rsvd[62]; } __packed req = { + .ver = 1, .idx = idx, .env = env_cap, .env_6g = dev->phy.power_type, .acpi_conf = mt792x_acpi_get_flags(&dev->phy), + .mtcl_conf = mt792x_acpi_get_mtcl_conf(&dev->phy, alpha2), }; int ret, valid_cnt = 0; - u8 i, *pos; + u16 buf_len = 0; + u8 *pos; if (!clc) return 0; @@ -1279,12 +1283,15 @@ int __mt7921_mcu_set_clc(struct mt792x_dev *dev, u8 *alpha2, if (mt76_find_power_limits_node(&dev->mt76)) req.cap |= CLC_CAP_DTS_EN; + buf_len = le16_to_cpu(clc->len) - sizeof(*clc); pos = clc->data; - for (i = 0; i < clc->nr_country; i++) { + while (buf_len > 16) { struct mt7921_clc_rule *rule = (struct mt7921_clc_rule *)pos; u16 len = le16_to_cpu(rule->len); + u16 offset = len + sizeof(*rule); - pos += len + sizeof(*rule); + pos += offset; + buf_len -= offset; if (rule->alpha2[0] != alpha2[0] || rule->alpha2[1] != alpha2[1]) continue; diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mt7921.h b/drivers/net/wireless/mediatek/mt76/mt7921/mt7921.h index f28621121927e5..1cb21133992b70 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/mt7921.h +++ b/drivers/net/wireless/mediatek/mt76/mt7921/mt7921.h @@ -12,7 +12,8 @@ #define MT7921_TX_FWDL_RING_SIZE 128 #define MT7921_RX_RING_SIZE 1536 -#define MT7921_RX_MCU_RING_SIZE 512 +#define MT7921_RX_MCU_RING_SIZE 8 +#define MT7921_RX_MCU_WA_RING_SIZE 512 #define MT7921_EEPROM_SIZE 3584 #define MT7921_TOKEN_SIZE 8192 @@ -233,6 +234,7 @@ mt7921_l1_rmw(struct mt792x_dev *dev, u32 addr, u32 mask, u32 val) #define mt7921_l1_set(dev, addr, val) mt7921_l1_rmw(dev, addr, 0, val) #define mt7921_l1_clear(dev, addr, val) mt7921_l1_rmw(dev, addr, val, 0) +void mt7921_regd_update(struct mt792x_dev *dev); int mt7921_mac_init(struct mt792x_dev *dev); bool mt7921_mac_wtbl_update(struct mt792x_dev *dev, int idx, u32 mask); int mt7921_mac_sta_add(struct mt76_dev *mdev, struct ieee80211_vif *vif, diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/pci.c b/drivers/net/wireless/mediatek/mt76/mt7921/pci.c index f04e7095e18109..57903c6e4f11f0 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/pci.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/pci.c @@ -171,7 +171,7 @@ static int mt7921_dma_init(struct mt792x_dev *dev) /* init tx queue */ ret = mt76_connac_init_tx_queues(dev->phy.mt76, MT7921_TXQ_BAND0, MT7921_TX_RING_SIZE, - MT_TX_RING_BASE, 0); + MT_TX_RING_BASE, NULL, 0); if (ret) return ret; @@ -200,7 +200,7 @@ static int mt7921_dma_init(struct mt792x_dev *dev) /* Change mcu queue after firmware download */ ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_MCU_WA], MT7921_RXQ_MCU_WM, - MT7921_RX_MCU_RING_SIZE, + MT7921_RX_MCU_WA_RING_SIZE, MT_RX_BUF_SIZE, MT_WFDMA0(0x540)); if (ret) return ret; @@ -507,6 +507,9 @@ static int mt7921_pci_resume(struct device *device) mt76_connac_mcu_set_deep_sleep(&dev->mt76, false); err = mt76_connac_mcu_set_hif_suspend(mdev, false); + + mt7921_regd_update(dev); + failed: pm->suspended = false; diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/sdio.c b/drivers/net/wireless/mediatek/mt76/mt7921/sdio.c index dc1beb76df3e16..7591e54d289733 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/sdio.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/sdio.c @@ -228,7 +228,7 @@ static int mt7921s_suspend(struct device *__dev) mt76_txq_schedule_all(&dev->mphy); mt76_worker_disable(&mdev->tx_worker); mt76_worker_disable(&mdev->sdio.status_worker); - cancel_work_sync(&mdev->sdio.stat_work); + mt76_worker_disable(&mdev->sdio.stat_worker); clear_bit(MT76_READING_STATS, &dev->mphy.state); mt76_tx_status_check(mdev, true); @@ -260,6 +260,7 @@ static int mt7921s_suspend(struct device *__dev) restore_worker: mt76_worker_enable(&mdev->tx_worker); mt76_worker_enable(&mdev->sdio.status_worker); + mt76_worker_enable(&mdev->sdio.stat_worker); if (!pm->ds_enable) mt76_connac_mcu_set_deep_sleep(mdev, false); @@ -292,6 +293,7 @@ static int mt7921s_resume(struct device *__dev) mt76_worker_enable(&mdev->sdio.txrx_worker); mt76_worker_enable(&mdev->sdio.status_worker); mt76_worker_enable(&mdev->sdio.net_worker); + mt76_worker_enable(&mdev->sdio.stat_worker); /* restore previous ds setting */ if (!pm->ds_enable) diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/sdio_mac.c b/drivers/net/wireless/mediatek/mt76/mt7921/sdio_mac.c index 8edd0291c12801..389eb0903807e9 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/sdio_mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/sdio_mac.c @@ -107,7 +107,7 @@ int mt7921s_mac_reset(struct mt792x_dev *dev) mt76_worker_disable(&dev->mt76.sdio.txrx_worker); mt76_worker_disable(&dev->mt76.sdio.status_worker); mt76_worker_disable(&dev->mt76.sdio.net_worker); - cancel_work_sync(&dev->mt76.sdio.stat_work); + mt76_worker_disable(&dev->mt76.sdio.stat_worker); mt7921s_disable_irq(&dev->mt76); mt7921s_wfsys_reset(dev); @@ -115,6 +115,7 @@ int mt7921s_mac_reset(struct mt792x_dev *dev) mt76_worker_enable(&dev->mt76.sdio.txrx_worker); mt76_worker_enable(&dev->mt76.sdio.status_worker); mt76_worker_enable(&dev->mt76.sdio.net_worker); + mt76_worker_enable(&dev->mt76.sdio.stat_worker); dev->fw_assert = false; clear_bit(MT76_MCU_RESET, &dev->mphy.state); diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/main.c b/drivers/net/wireless/mediatek/mt76/mt7925/main.c index aa918b9b0469f8..8f1075da490390 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/main.c @@ -154,8 +154,7 @@ mt7925_init_he_caps(struct mt792x_phy *phy, enum nl80211_band band, static void mt7925_init_eht_caps(struct mt792x_phy *phy, enum nl80211_band band, - struct ieee80211_sband_iftype_data *data, - enum nl80211_iftype iftype) + struct ieee80211_sband_iftype_data *data) { struct ieee80211_sta_eht_cap *eht_cap = &data->eht_cap; struct ieee80211_eht_cap_elem_fixed *eht_cap_elem = &eht_cap->eht_cap_elem; @@ -256,7 +255,7 @@ __mt7925_set_stream_he_eht_caps(struct mt792x_phy *phy, data[n].types_mask = BIT(i); mt7925_init_he_caps(phy, band, &data[n], i); - mt7925_init_eht_caps(phy, band, &data[n], i); + mt7925_init_eht_caps(phy, band, &data[n]); n++; } diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c index 9c0e397537acf5..c5fd7116929b74 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c @@ -345,7 +345,7 @@ static void mt7925_mcu_scan_event(struct mt792x_dev *dev, struct sk_buff *skb) { struct mt76_phy *mphy = &dev->mt76.phy; - struct mt792x_phy *phy = (struct mt792x_phy *)mphy->priv; + struct mt792x_phy *phy = mphy->priv; spin_lock_bh(&dev->mt76.lock); __skb_queue_tail(&phy->scan_event_list, skb); diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/pci.c b/drivers/net/wireless/mediatek/mt76/mt7925/pci.c index 08ef75e24e1cf6..734f31ee40d3f7 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/pci.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/pci.c @@ -218,7 +218,7 @@ static int mt7925_dma_init(struct mt792x_dev *dev) /* init tx queue */ ret = mt76_connac_init_tx_queues(dev->phy.mt76, MT7925_TXQ_BAND0, MT7925_TX_RING_SIZE, - MT_TX_RING_BASE, 0); + MT_TX_RING_BASE, NULL, 0); if (ret) return ret; diff --git a/drivers/net/wireless/mediatek/mt76/mt792x.h b/drivers/net/wireless/mediatek/mt76/mt792x.h index 36fae736dd1925..3c897b34aaa796 100644 --- a/drivers/net/wireless/mediatek/mt76/mt792x.h +++ b/drivers/net/wireless/mediatek/mt76/mt792x.h @@ -382,6 +382,7 @@ int mt792xe_mcu_fw_pmctrl(struct mt792x_dev *dev); int mt792x_init_acpi_sar(struct mt792x_dev *dev); int mt792x_init_acpi_sar_power(struct mt792x_phy *phy, bool set_default); u8 mt792x_acpi_get_flags(struct mt792x_phy *phy); +u8 mt792x_acpi_get_mtcl_conf(struct mt792x_phy *phy, char *alpha2); #else static inline int mt792x_init_acpi_sar(struct mt792x_dev *dev) { @@ -398,6 +399,11 @@ static inline u8 mt792x_acpi_get_flags(struct mt792x_phy *phy) { return 0; } + +static inline u8 mt792x_acpi_get_mtcl_conf(struct mt792x_phy *phy, char *alpha2) +{ + return 0xf; +} #endif #endif /* __MT7925_H */ diff --git a/drivers/net/wireless/mediatek/mt76/mt792x_acpi_sar.c b/drivers/net/wireless/mediatek/mt76/mt792x_acpi_sar.c index 303c0f5c9c662c..e7afea87e82e2b 100644 --- a/drivers/net/wireless/mediatek/mt76/mt792x_acpi_sar.c +++ b/drivers/net/wireless/mediatek/mt76/mt792x_acpi_sar.c @@ -348,3 +348,56 @@ u8 mt792x_acpi_get_flags(struct mt792x_phy *phy) return flags; } EXPORT_SYMBOL_GPL(mt792x_acpi_get_flags); + +static u8 +mt792x_acpi_get_mtcl_map(int row, int column, struct mt792x_asar_cl *cl) +{ + u8 config = 0; + + if (cl->cl6g[row] & BIT(column)) + config |= (cl->mode_6g & 0x3) << 2; + if (cl->version > 1 && cl->cl5g9[row] & BIT(column)) + config |= (cl->mode_5g9 & 0x3); + + return config; +} + +u8 mt792x_acpi_get_mtcl_conf(struct mt792x_phy *phy, char *alpha2) +{ + static const char * const cc_list_all[] = { + "00", "EU", "AR", "AU", "AZ", "BY", "BO", "BR", + "CA", "CL", "CN", "ID", "JP", "MY", "MX", "ME", + "MA", "NZ", "NG", "PH", "RU", "RS", "SG", "KR", + "TW", "TH", "UA", "GB", "US", "VN", "KH", "PY", + }; + static const char * const cc_list_eu[] = { + "AT", "BE", "BG", "CY", "CZ", "HR", "DK", "EE", + "FI", "FR", "DE", "GR", "HU", "IS", "IE", "IT", + "LV", "LI", "LT", "LU", "MT", "NL", "NO", "PL", + "PT", "RO", "MT", "SK", "SI", "ES", "CH", + }; + struct mt792x_acpi_sar *sar = phy->acpisar; + struct mt792x_asar_cl *cl; + int col, row, i; + + if (!sar) + return 0xf; + + cl = sar->countrylist; + if (!cl) + return 0xc; + + for (i = 0; i < ARRAY_SIZE(cc_list_all); i++) { + col = 7 - i % 8; + row = i / 8; + if (!memcmp(cc_list_all[i], alpha2, 2)) + return mt792x_acpi_get_mtcl_map(row, col, cl); + } + + for (i = 0; i < ARRAY_SIZE(cc_list_eu); i++) + if (!memcmp(cc_list_eu[i], alpha2, 2)) + return mt792x_acpi_get_mtcl_map(0, 6, cl); + + return mt792x_acpi_get_mtcl_map(0, 7, cl); +} +EXPORT_SYMBOL_GPL(mt792x_acpi_get_mtcl_conf); diff --git a/drivers/net/wireless/mediatek/mt76/mt792x_acpi_sar.h b/drivers/net/wireless/mediatek/mt76/mt792x_acpi_sar.h index d6d332e863ba9c..2298983b6342fd 100644 --- a/drivers/net/wireless/mediatek/mt76/mt792x_acpi_sar.h +++ b/drivers/net/wireless/mediatek/mt76/mt792x_acpi_sar.h @@ -77,6 +77,8 @@ struct mt792x_asar_cl { u8 version; u8 mode_6g; u8 cl6g[6]; + u8 mode_5g9; + u8 cl5g9[6]; } __packed; struct mt792x_asar_fg { diff --git a/drivers/net/wireless/mediatek/mt76/mt792x_mac.c b/drivers/net/wireless/mediatek/mt76/mt792x_mac.c index 5d1f8229fdc1da..eb29434abee1bc 100644 --- a/drivers/net/wireless/mediatek/mt76/mt792x_mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt792x_mac.c @@ -223,7 +223,7 @@ static void mt792x_phy_update_channel(struct mt76_phy *mphy, int idx) { struct mt792x_dev *dev = container_of(mphy->dev, struct mt792x_dev, mt76); - struct mt792x_phy *phy = (struct mt792x_phy *)mphy->priv; + struct mt792x_phy *phy = mphy->priv; struct mt76_channel_state *state; u64 busy_time, tx_time, rx_time, obss_time; int nf; diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/debugfs.c b/drivers/net/wireless/mediatek/mt76/mt7996/debugfs.c index 4d40ec7ff57f53..9bd953586b0411 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/debugfs.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/debugfs.c @@ -476,7 +476,7 @@ mt7996_txbf_stat_read_phy(struct mt7996_phy *phy, struct seq_file *s) { struct mt76_mib_stats *mib = &phy->mib; static const char * const bw[] = { - "BW20", "BW40", "BW80", "BW160" + "BW20", "BW40", "BW80", "BW160", "BW320" }; /* Tx Beamformer monitor */ @@ -489,8 +489,9 @@ mt7996_txbf_stat_read_phy(struct mt7996_phy *phy, struct seq_file *s) /* Tx Beamformer Rx feedback monitor */ seq_puts(s, "Tx Beamformer Rx feedback statistics: "); - seq_printf(s, "All: %d, HE: %d, VHT: %d, HT: %d, ", + seq_printf(s, "All: %d, EHT: %d, HE: %d, VHT: %d, HT: %d, ", mib->tx_bf_rx_fb_all_cnt, + mib->tx_bf_rx_fb_eht_cnt, mib->tx_bf_rx_fb_he_cnt, mib->tx_bf_rx_fb_vht_cnt, mib->tx_bf_rx_fb_ht_cnt); diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/dma.c b/drivers/net/wireless/mediatek/mt76/mt7996/dma.c index 586e247a1e0648..483ad81b6eec63 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/dma.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/dma.c @@ -7,6 +7,26 @@ #include "../dma.h" #include "mac.h" +int mt7996_init_tx_queues(struct mt7996_phy *phy, int idx, int n_desc, + int ring_base, struct mtk_wed_device *wed) +{ + struct mt7996_dev *dev = phy->dev; + u32 flags = 0; + + if (mtk_wed_device_active(wed)) { + ring_base += MT_TXQ_ID(0) * MT_RING_SIZE; + idx -= MT_TXQ_ID(0); + + if (phy->mt76->band_idx == MT_BAND2) + flags = MT_WED_Q_TX(0); + else + flags = MT_WED_Q_TX(idx); + } + + return mt76_connac_init_tx_queues(phy->mt76, idx, n_desc, + ring_base, wed, flags); +} + static int mt7996_poll_tx(struct napi_struct *napi, int budget) { struct mt7996_dev *dev; @@ -37,18 +57,51 @@ static void mt7996_dma_config(struct mt7996_dev *dev) RXQ_CONFIG(MT_RXQ_MCU, WFDMA0, MT_INT_RX_DONE_WM, MT7996_RXQ_MCU_WM); RXQ_CONFIG(MT_RXQ_MCU_WA, WFDMA0, MT_INT_RX_DONE_WA, MT7996_RXQ_MCU_WA); - /* band0/band1 */ + /* mt7996: band0 and band1, mt7992: band0 */ RXQ_CONFIG(MT_RXQ_MAIN, WFDMA0, MT_INT_RX_DONE_BAND0, MT7996_RXQ_BAND0); RXQ_CONFIG(MT_RXQ_MAIN_WA, WFDMA0, MT_INT_RX_DONE_WA_MAIN, MT7996_RXQ_MCU_WA_MAIN); - /* band2 */ - RXQ_CONFIG(MT_RXQ_BAND2, WFDMA0, MT_INT_RX_DONE_BAND2, MT7996_RXQ_BAND2); - RXQ_CONFIG(MT_RXQ_BAND2_WA, WFDMA0, MT_INT_RX_DONE_WA_TRI, MT7996_RXQ_MCU_WA_TRI); + if (is_mt7996(&dev->mt76)) { + /* mt7996 band2 */ + RXQ_CONFIG(MT_RXQ_BAND2, WFDMA0, MT_INT_RX_DONE_BAND2, MT7996_RXQ_BAND2); + RXQ_CONFIG(MT_RXQ_BAND2_WA, WFDMA0, MT_INT_RX_DONE_WA_TRI, MT7996_RXQ_MCU_WA_TRI); + } else { + /* mt7992 band1 */ + RXQ_CONFIG(MT_RXQ_BAND1, WFDMA0, MT_INT_RX_DONE_BAND1, MT7996_RXQ_BAND1); + RXQ_CONFIG(MT_RXQ_BAND1_WA, WFDMA0, MT_INT_RX_DONE_WA_EXT, MT7996_RXQ_MCU_WA_EXT); + } + + if (dev->has_rro) { + /* band0 */ + RXQ_CONFIG(MT_RXQ_RRO_BAND0, WFDMA0, MT_INT_RX_DONE_RRO_BAND0, + MT7996_RXQ_RRO_BAND0); + RXQ_CONFIG(MT_RXQ_MSDU_PAGE_BAND0, WFDMA0, MT_INT_RX_DONE_MSDU_PG_BAND0, + MT7996_RXQ_MSDU_PG_BAND0); + RXQ_CONFIG(MT_RXQ_TXFREE_BAND0, WFDMA0, MT_INT_RX_TXFREE_MAIN, + MT7996_RXQ_TXFREE0); + /* band1 */ + RXQ_CONFIG(MT_RXQ_MSDU_PAGE_BAND1, WFDMA0, MT_INT_RX_DONE_MSDU_PG_BAND1, + MT7996_RXQ_MSDU_PG_BAND1); + /* band2 */ + RXQ_CONFIG(MT_RXQ_RRO_BAND2, WFDMA0, MT_INT_RX_DONE_RRO_BAND2, + MT7996_RXQ_RRO_BAND2); + RXQ_CONFIG(MT_RXQ_MSDU_PAGE_BAND2, WFDMA0, MT_INT_RX_DONE_MSDU_PG_BAND2, + MT7996_RXQ_MSDU_PG_BAND2); + RXQ_CONFIG(MT_RXQ_TXFREE_BAND2, WFDMA0, MT_INT_RX_TXFREE_TRI, + MT7996_RXQ_TXFREE2); + + RXQ_CONFIG(MT_RXQ_RRO_IND, WFDMA0, MT_INT_RX_DONE_RRO_IND, + MT7996_RXQ_RRO_IND); + } /* data tx queue */ TXQ_CONFIG(0, WFDMA0, MT_INT_TX_DONE_BAND0, MT7996_TXQ_BAND0); - TXQ_CONFIG(1, WFDMA0, MT_INT_TX_DONE_BAND1, MT7996_TXQ_BAND1); - TXQ_CONFIG(2, WFDMA0, MT_INT_TX_DONE_BAND2, MT7996_TXQ_BAND2); + if (is_mt7996(&dev->mt76)) { + TXQ_CONFIG(1, WFDMA0, MT_INT_TX_DONE_BAND1, MT7996_TXQ_BAND1); + TXQ_CONFIG(2, WFDMA0, MT_INT_TX_DONE_BAND2, MT7996_TXQ_BAND2); + } else { + TXQ_CONFIG(1, WFDMA0, MT_INT_TX_DONE_BAND1, MT7996_TXQ_BAND1); + } /* mcu tx queue */ MCUQ_CONFIG(MT_MCUQ_WM, WFDMA0, MT_INT_TX_DONE_MCU_WM, MT7996_TXQ_MCU_WM); @@ -56,22 +109,57 @@ static void mt7996_dma_config(struct mt7996_dev *dev) MCUQ_CONFIG(MT_MCUQ_FWDL, WFDMA0, MT_INT_TX_DONE_FWDL, MT7996_TXQ_FWDL); } +static u32 __mt7996_dma_prefetch_base(u16 *base, u8 depth) +{ + u32 ret = *base << 16 | depth; + + *base = *base + (depth << 4); + + return ret; +} + static void __mt7996_dma_prefetch(struct mt7996_dev *dev, u32 ofs) { -#define PREFETCH(_base, _depth) ((_base) << 16 | (_depth)) + u16 base = 0; + u8 queue; + +#define PREFETCH(_depth) (__mt7996_dma_prefetch_base(&base, (_depth))) /* prefetch SRAM wrapping boundary for tx/rx ring. */ - mt76_wr(dev, MT_MCUQ_EXT_CTRL(MT_MCUQ_FWDL) + ofs, PREFETCH(0x0, 0x2)); - mt76_wr(dev, MT_MCUQ_EXT_CTRL(MT_MCUQ_WM) + ofs, PREFETCH(0x20, 0x2)); - mt76_wr(dev, MT_TXQ_EXT_CTRL(0) + ofs, PREFETCH(0x40, 0x4)); - mt76_wr(dev, MT_TXQ_EXT_CTRL(1) + ofs, PREFETCH(0x80, 0x4)); - mt76_wr(dev, MT_MCUQ_EXT_CTRL(MT_MCUQ_WA) + ofs, PREFETCH(0xc0, 0x2)); - mt76_wr(dev, MT_TXQ_EXT_CTRL(2) + ofs, PREFETCH(0xe0, 0x4)); - mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_MCU) + ofs, PREFETCH(0x120, 0x2)); - mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_MCU_WA) + ofs, PREFETCH(0x140, 0x2)); - mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_MAIN_WA) + ofs, PREFETCH(0x160, 0x2)); - mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_BAND2_WA) + ofs, PREFETCH(0x180, 0x2)); - mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_MAIN) + ofs, PREFETCH(0x1a0, 0x10)); - mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_BAND2) + ofs, PREFETCH(0x2a0, 0x10)); + mt76_wr(dev, MT_MCUQ_EXT_CTRL(MT_MCUQ_FWDL) + ofs, PREFETCH(0x2)); + mt76_wr(dev, MT_MCUQ_EXT_CTRL(MT_MCUQ_WM) + ofs, PREFETCH(0x2)); + mt76_wr(dev, MT_TXQ_EXT_CTRL(0) + ofs, PREFETCH(0x8)); + mt76_wr(dev, MT_TXQ_EXT_CTRL(1) + ofs, PREFETCH(0x8)); + mt76_wr(dev, MT_MCUQ_EXT_CTRL(MT_MCUQ_WA) + ofs, PREFETCH(0x2)); + mt76_wr(dev, MT_TXQ_EXT_CTRL(2) + ofs, PREFETCH(0x8)); + mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_MCU) + ofs, PREFETCH(0x2)); + mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_MCU_WA) + ofs, PREFETCH(0x2)); + mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_MAIN_WA) + ofs, PREFETCH(0x2)); + + queue = is_mt7996(&dev->mt76) ? MT_RXQ_BAND2_WA : MT_RXQ_BAND1_WA; + mt76_wr(dev, MT_RXQ_BAND1_CTRL(queue) + ofs, PREFETCH(0x2)); + + mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_MAIN) + ofs, PREFETCH(0x10)); + + queue = is_mt7996(&dev->mt76) ? MT_RXQ_BAND2 : MT_RXQ_BAND1; + mt76_wr(dev, MT_RXQ_BAND1_CTRL(queue) + ofs, PREFETCH(0x10)); + + if (dev->has_rro) { + mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_RRO_BAND0) + ofs, + PREFETCH(0x10)); + mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_RRO_BAND2) + ofs, + PREFETCH(0x10)); + mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_MSDU_PAGE_BAND0) + ofs, + PREFETCH(0x4)); + mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_MSDU_PAGE_BAND1) + ofs, + PREFETCH(0x4)); + mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_MSDU_PAGE_BAND2) + ofs, + PREFETCH(0x4)); + mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_TXFREE_BAND0) + ofs, + PREFETCH(0x4)); + mt76_wr(dev, MT_RXQ_BAND1_CTRL(MT_RXQ_TXFREE_BAND2) + ofs, + PREFETCH(0x4)); + } +#undef PREFETCH mt76_set(dev, WF_WFDMA0_GLO_CFG_EXT1 + ofs, WF_WFDMA0_GLO_CFG_EXT1_CALC_MODE); } @@ -128,8 +216,9 @@ static void mt7996_dma_disable(struct mt7996_dev *dev, bool reset) } } -void mt7996_dma_start(struct mt7996_dev *dev, bool reset) +void mt7996_dma_start(struct mt7996_dev *dev, bool reset, bool wed_reset) { + struct mtk_wed_device *wed = &dev->mt76.mmio.wed; u32 hif1_ofs = 0; u32 irq_mask; @@ -138,37 +227,49 @@ void mt7996_dma_start(struct mt7996_dev *dev, bool reset) /* enable WFDMA Tx/Rx */ if (!reset) { - mt76_set(dev, MT_WFDMA0_GLO_CFG, - MT_WFDMA0_GLO_CFG_TX_DMA_EN | - MT_WFDMA0_GLO_CFG_RX_DMA_EN | - MT_WFDMA0_GLO_CFG_OMIT_TX_INFO | - MT_WFDMA0_GLO_CFG_OMIT_RX_INFO_PFET2); + if (mtk_wed_device_active(wed) && mtk_wed_get_rx_capa(wed)) + mt76_set(dev, MT_WFDMA0_GLO_CFG, + MT_WFDMA0_GLO_CFG_TX_DMA_EN | + MT_WFDMA0_GLO_CFG_OMIT_TX_INFO | + MT_WFDMA0_GLO_CFG_EXT_EN); + else + mt76_set(dev, MT_WFDMA0_GLO_CFG, + MT_WFDMA0_GLO_CFG_TX_DMA_EN | + MT_WFDMA0_GLO_CFG_RX_DMA_EN | + MT_WFDMA0_GLO_CFG_OMIT_TX_INFO | + MT_WFDMA0_GLO_CFG_OMIT_RX_INFO_PFET2); if (dev->hif2) mt76_set(dev, MT_WFDMA0_GLO_CFG + hif1_ofs, MT_WFDMA0_GLO_CFG_TX_DMA_EN | MT_WFDMA0_GLO_CFG_RX_DMA_EN | MT_WFDMA0_GLO_CFG_OMIT_TX_INFO | - MT_WFDMA0_GLO_CFG_OMIT_RX_INFO_PFET2); + MT_WFDMA0_GLO_CFG_OMIT_RX_INFO_PFET2 | + MT_WFDMA0_GLO_CFG_EXT_EN); } /* enable interrupts for TX/RX rings */ - irq_mask = MT_INT_MCU_CMD; - if (reset) - goto done; + irq_mask = MT_INT_MCU_CMD | MT_INT_RX_DONE_MCU | MT_INT_TX_DONE_MCU; - irq_mask = MT_INT_RX_DONE_MCU | MT_INT_TX_DONE_MCU; - - if (!dev->mphy.band_idx) + if (mt7996_band_valid(dev, MT_BAND0)) irq_mask |= MT_INT_BAND0_RX_DONE; - if (dev->dbdc_support) + if (mt7996_band_valid(dev, MT_BAND1)) irq_mask |= MT_INT_BAND1_RX_DONE; - if (dev->tbtc_support) + if (mt7996_band_valid(dev, MT_BAND2)) irq_mask |= MT_INT_BAND2_RX_DONE; -done: + if (mtk_wed_device_active(wed) && wed_reset) { + u32 wed_irq_mask = irq_mask; + + wed_irq_mask |= MT_INT_TX_DONE_BAND0 | MT_INT_TX_DONE_BAND1; + mt76_wr(dev, MT_INT_MASK_CSR, wed_irq_mask); + mtk_wed_device_start(wed, wed_irq_mask); + } + + irq_mask = reset ? MT_INT_MCU_CMD : irq_mask; + mt7996_irq_enable(dev, irq_mask); mt7996_irq_disable(dev, 0); } @@ -223,6 +324,12 @@ static void mt7996_dma_enable(struct mt7996_dev *dev, bool reset) mt76_set(dev, WF_WFDMA0_GLO_CFG_EXT1, WF_WFDMA0_GLO_CFG_EXT1_TX_FCTRL_MODE); + /* WFDMA rx threshold */ + mt76_wr(dev, MT_WFDMA0_PAUSE_RX_Q_45_TH, 0xc000c); + mt76_wr(dev, MT_WFDMA0_PAUSE_RX_Q_67_TH, 0x10008); + mt76_wr(dev, MT_WFDMA0_PAUSE_RX_Q_89_TH, 0x10008); + mt76_wr(dev, MT_WFDMA0_PAUSE_RX_Q_RRO_TH, 0x20); + if (dev->hif2) { /* GLO_CFG_EXT0 */ mt76_set(dev, WF_WFDMA0_GLO_CFG_EXT0 + hif1_ofs, @@ -234,24 +341,108 @@ static void mt7996_dma_enable(struct mt7996_dev *dev, bool reset) WF_WFDMA0_GLO_CFG_EXT1_TX_FCTRL_MODE); mt76_set(dev, MT_WFDMA_HOST_CONFIG, - MT_WFDMA_HOST_CONFIG_PDMA_BAND); + MT_WFDMA_HOST_CONFIG_PDMA_BAND | + MT_WFDMA_HOST_CONFIG_BAND2_PCIE1); + + /* AXI read outstanding number */ + mt76_rmw(dev, MT_WFDMA_AXI_R2A_CTRL, + MT_WFDMA_AXI_R2A_CTRL_OUTSTAND_MASK, 0x14); + + /* WFDMA rx threshold */ + mt76_wr(dev, MT_WFDMA0_PAUSE_RX_Q_45_TH + hif1_ofs, 0xc000c); + mt76_wr(dev, MT_WFDMA0_PAUSE_RX_Q_67_TH + hif1_ofs, 0x10008); + mt76_wr(dev, MT_WFDMA0_PAUSE_RX_Q_89_TH + hif1_ofs, 0x10008); + mt76_wr(dev, MT_WFDMA0_PAUSE_RX_Q_RRO_TH + hif1_ofs, 0x20); } if (dev->hif2) { /* fix hardware limitation, pcie1's rx ring3 is not available * so, redirect pcie0 rx ring3 interrupt to pcie1 */ - mt76_set(dev, MT_WFDMA0_RX_INT_PCIE_SEL, - MT_WFDMA0_RX_INT_SEL_RING3); + if (mtk_wed_device_active(&dev->mt76.mmio.wed) && + dev->has_rro) + mt76_set(dev, MT_WFDMA0_RX_INT_PCIE_SEL + hif1_ofs, + MT_WFDMA0_RX_INT_SEL_RING6); + else + mt76_set(dev, MT_WFDMA0_RX_INT_PCIE_SEL, + MT_WFDMA0_RX_INT_SEL_RING3); + } + + mt7996_dma_start(dev, reset, true); +} + +#ifdef CONFIG_NET_MEDIATEK_SOC_WED +int mt7996_dma_rro_init(struct mt7996_dev *dev) +{ + struct mt76_dev *mdev = &dev->mt76; + u32 irq_mask; + int ret; - /* TODO: redirect rx ring6 interrupt to pcie0 for wed function */ + /* ind cmd */ + mdev->q_rx[MT_RXQ_RRO_IND].flags = MT_WED_RRO_Q_IND; + mdev->q_rx[MT_RXQ_RRO_IND].wed = &mdev->mmio.wed; + ret = mt76_queue_alloc(dev, &mdev->q_rx[MT_RXQ_RRO_IND], + MT_RXQ_ID(MT_RXQ_RRO_IND), + MT7996_RX_RING_SIZE, + 0, MT_RXQ_RRO_IND_RING_BASE); + if (ret) + return ret; + + /* rx msdu page queue for band0 */ + mdev->q_rx[MT_RXQ_MSDU_PAGE_BAND0].flags = + MT_WED_RRO_Q_MSDU_PG(0) | MT_QFLAG_WED_RRO_EN; + mdev->q_rx[MT_RXQ_MSDU_PAGE_BAND0].wed = &mdev->mmio.wed; + ret = mt76_queue_alloc(dev, &mdev->q_rx[MT_RXQ_MSDU_PAGE_BAND0], + MT_RXQ_ID(MT_RXQ_MSDU_PAGE_BAND0), + MT7996_RX_RING_SIZE, + MT7996_RX_MSDU_PAGE_SIZE, + MT_RXQ_RING_BASE(MT_RXQ_MSDU_PAGE_BAND0)); + if (ret) + return ret; + + if (mt7996_band_valid(dev, MT_BAND1)) { + /* rx msdu page queue for band1 */ + mdev->q_rx[MT_RXQ_MSDU_PAGE_BAND1].flags = + MT_WED_RRO_Q_MSDU_PG(1) | MT_QFLAG_WED_RRO_EN; + mdev->q_rx[MT_RXQ_MSDU_PAGE_BAND1].wed = &mdev->mmio.wed; + ret = mt76_queue_alloc(dev, &mdev->q_rx[MT_RXQ_MSDU_PAGE_BAND1], + MT_RXQ_ID(MT_RXQ_MSDU_PAGE_BAND1), + MT7996_RX_RING_SIZE, + MT7996_RX_MSDU_PAGE_SIZE, + MT_RXQ_RING_BASE(MT_RXQ_MSDU_PAGE_BAND1)); + if (ret) + return ret; } - mt7996_dma_start(dev, reset); + if (mt7996_band_valid(dev, MT_BAND2)) { + /* rx msdu page queue for band2 */ + mdev->q_rx[MT_RXQ_MSDU_PAGE_BAND2].flags = + MT_WED_RRO_Q_MSDU_PG(2) | MT_QFLAG_WED_RRO_EN; + mdev->q_rx[MT_RXQ_MSDU_PAGE_BAND2].wed = &mdev->mmio.wed; + ret = mt76_queue_alloc(dev, &mdev->q_rx[MT_RXQ_MSDU_PAGE_BAND2], + MT_RXQ_ID(MT_RXQ_MSDU_PAGE_BAND2), + MT7996_RX_RING_SIZE, + MT7996_RX_MSDU_PAGE_SIZE, + MT_RXQ_RING_BASE(MT_RXQ_MSDU_PAGE_BAND2)); + if (ret) + return ret; + } + + irq_mask = mdev->mmio.irqmask | MT_INT_RRO_RX_DONE | + MT_INT_TX_DONE_BAND2; + mt76_wr(dev, MT_INT_MASK_CSR, irq_mask); + mtk_wed_device_start_hw_rro(&mdev->mmio.wed, irq_mask, false); + mt7996_irq_enable(dev, irq_mask); + + return 0; } +#endif /* CONFIG_NET_MEDIATEK_SOC_WED */ int mt7996_dma_init(struct mt7996_dev *dev) { + struct mtk_wed_device *wed = &dev->mt76.mmio.wed; + struct mtk_wed_device *wed_hif2 = &dev->mt76.mmio.wed_hif2; + u32 rx_base; u32 hif1_ofs = 0; int ret; @@ -265,10 +456,11 @@ int mt7996_dma_init(struct mt7996_dev *dev) mt7996_dma_disable(dev, true); /* init tx queue */ - ret = mt76_connac_init_tx_queues(dev->phy.mt76, - MT_TXQ_ID(dev->mphy.band_idx), - MT7996_TX_RING_SIZE, - MT_TXQ_RING_BASE(0), 0); + ret = mt7996_init_tx_queues(&dev->phy, + MT_TXQ_ID(dev->mphy.band_idx), + MT7996_TX_RING_SIZE, + MT_TXQ_RING_BASE(0), + wed); if (ret) return ret; @@ -314,7 +506,12 @@ int mt7996_dma_init(struct mt7996_dev *dev) if (ret) return ret; - /* rx data queue for band0 and band1 */ + /* rx data queue for band0 and mt7996 band1 */ + if (mtk_wed_device_active(wed) && mtk_wed_get_rx_capa(wed)) { + dev->mt76.q_rx[MT_RXQ_MAIN].flags = MT_WED_Q_RX(0); + dev->mt76.q_rx[MT_RXQ_MAIN].wed = wed; + } + ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_MAIN], MT_RXQ_ID(MT_RXQ_MAIN), MT7996_RX_RING_SIZE, @@ -324,6 +521,11 @@ int mt7996_dma_init(struct mt7996_dev *dev) return ret; /* tx free notify event from WA for band0 */ + if (mtk_wed_device_active(wed) && !dev->has_rro) { + dev->mt76.q_rx[MT_RXQ_MAIN_WA].flags = MT_WED_Q_TXFREE; + dev->mt76.q_rx[MT_RXQ_MAIN_WA].wed = wed; + } + ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_MAIN_WA], MT_RXQ_ID(MT_RXQ_MAIN_WA), MT7996_RX_MCU_RING_SIZE, @@ -332,19 +534,25 @@ int mt7996_dma_init(struct mt7996_dev *dev) if (ret) return ret; - if (dev->tbtc_support || dev->mphy.band_idx == MT_BAND2) { - /* rx data queue for band2 */ + if (mt7996_band_valid(dev, MT_BAND2)) { + /* rx data queue for mt7996 band2 */ + rx_base = MT_RXQ_RING_BASE(MT_RXQ_BAND2) + hif1_ofs; ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_BAND2], MT_RXQ_ID(MT_RXQ_BAND2), MT7996_RX_RING_SIZE, MT_RX_BUF_SIZE, - MT_RXQ_RING_BASE(MT_RXQ_BAND2) + hif1_ofs); + rx_base); if (ret) return ret; - /* tx free notify event from WA for band2 + /* tx free notify event from WA for mt7996 band2 * use pcie0's rx ring3, but, redirect pcie0 rx ring3 interrupt to pcie1 */ + if (mtk_wed_device_active(wed_hif2) && !dev->has_rro) { + dev->mt76.q_rx[MT_RXQ_BAND2_WA].flags = MT_WED_Q_TXFREE; + dev->mt76.q_rx[MT_RXQ_BAND2_WA].wed = wed_hif2; + } + ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_BAND2_WA], MT_RXQ_ID(MT_RXQ_BAND2_WA), MT7996_RX_MCU_RING_SIZE, @@ -352,6 +560,80 @@ int mt7996_dma_init(struct mt7996_dev *dev) MT_RXQ_RING_BASE(MT_RXQ_BAND2_WA)); if (ret) return ret; + } else if (mt7996_band_valid(dev, MT_BAND1)) { + /* rx data queue for mt7992 band1 */ + rx_base = MT_RXQ_RING_BASE(MT_RXQ_BAND1) + hif1_ofs; + ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_BAND1], + MT_RXQ_ID(MT_RXQ_BAND1), + MT7996_RX_RING_SIZE, + MT_RX_BUF_SIZE, + rx_base); + if (ret) + return ret; + + /* tx free notify event from WA for mt7992 band1 */ + rx_base = MT_RXQ_RING_BASE(MT_RXQ_BAND1_WA) + hif1_ofs; + ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_BAND1_WA], + MT_RXQ_ID(MT_RXQ_BAND1_WA), + MT7996_RX_MCU_RING_SIZE, + MT_RX_BUF_SIZE, + rx_base); + if (ret) + return ret; + } + + if (mtk_wed_device_active(wed) && mtk_wed_get_rx_capa(wed) && + dev->has_rro) { + /* rx rro data queue for band0 */ + dev->mt76.q_rx[MT_RXQ_RRO_BAND0].flags = + MT_WED_RRO_Q_DATA(0) | MT_QFLAG_WED_RRO_EN; + dev->mt76.q_rx[MT_RXQ_RRO_BAND0].wed = wed; + ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_RRO_BAND0], + MT_RXQ_ID(MT_RXQ_RRO_BAND0), + MT7996_RX_RING_SIZE, + MT7996_RX_BUF_SIZE, + MT_RXQ_RING_BASE(MT_RXQ_RRO_BAND0)); + if (ret) + return ret; + + /* tx free notify event from WA for band0 */ + dev->mt76.q_rx[MT_RXQ_TXFREE_BAND0].flags = MT_WED_Q_TXFREE; + dev->mt76.q_rx[MT_RXQ_TXFREE_BAND0].wed = wed; + + ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_TXFREE_BAND0], + MT_RXQ_ID(MT_RXQ_TXFREE_BAND0), + MT7996_RX_MCU_RING_SIZE, + MT7996_RX_BUF_SIZE, + MT_RXQ_RING_BASE(MT_RXQ_TXFREE_BAND0)); + if (ret) + return ret; + + if (mt7996_band_valid(dev, MT_BAND2)) { + /* rx rro data queue for band2 */ + dev->mt76.q_rx[MT_RXQ_RRO_BAND2].flags = + MT_WED_RRO_Q_DATA(1) | MT_QFLAG_WED_RRO_EN; + dev->mt76.q_rx[MT_RXQ_RRO_BAND2].wed = wed; + ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_RRO_BAND2], + MT_RXQ_ID(MT_RXQ_RRO_BAND2), + MT7996_RX_RING_SIZE, + MT7996_RX_BUF_SIZE, + MT_RXQ_RING_BASE(MT_RXQ_RRO_BAND2) + hif1_ofs); + if (ret) + return ret; + + /* tx free notify event from MAC for band2 */ + if (mtk_wed_device_active(wed_hif2)) { + dev->mt76.q_rx[MT_RXQ_TXFREE_BAND2].flags = MT_WED_Q_TXFREE; + dev->mt76.q_rx[MT_RXQ_TXFREE_BAND2].wed = wed_hif2; + } + ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_TXFREE_BAND2], + MT_RXQ_ID(MT_RXQ_TXFREE_BAND2), + MT7996_RX_MCU_RING_SIZE, + MT7996_RX_BUF_SIZE, + MT_RXQ_RING_BASE(MT_RXQ_TXFREE_BAND2) + hif1_ofs); + if (ret) + return ret; + } } ret = mt76_init_queues(dev, mt76_dma_rx_poll); @@ -405,21 +687,33 @@ void mt7996_dma_reset(struct mt7996_dev *dev, bool force) if (force) mt7996_wfsys_reset(dev); + if (dev->hif2 && mtk_wed_device_active(&dev->mt76.mmio.wed_hif2)) + mtk_wed_device_dma_reset(&dev->mt76.mmio.wed_hif2); + + if (mtk_wed_device_active(&dev->mt76.mmio.wed)) + mtk_wed_device_dma_reset(&dev->mt76.mmio.wed); + mt7996_dma_disable(dev, force); + mt76_dma_wed_reset(&dev->mt76); /* reset hw queues */ for (i = 0; i < __MT_TXQ_MAX; i++) { - mt76_queue_reset(dev, dev->mphy.q_tx[i]); + mt76_dma_reset_tx_queue(&dev->mt76, dev->mphy.q_tx[i]); if (phy2) - mt76_queue_reset(dev, phy2->q_tx[i]); + mt76_dma_reset_tx_queue(&dev->mt76, phy2->q_tx[i]); if (phy3) - mt76_queue_reset(dev, phy3->q_tx[i]); + mt76_dma_reset_tx_queue(&dev->mt76, phy3->q_tx[i]); } for (i = 0; i < __MT_MCUQ_MAX; i++) mt76_queue_reset(dev, dev->mt76.q_mcu[i]); mt76_for_each_q_rx(&dev->mt76, i) { + if (mtk_wed_device_active(&dev->mt76.mmio.wed)) + if (mt76_queue_is_wed_rro(&dev->mt76.q_rx[i]) || + mt76_queue_is_wed_tx_free(&dev->mt76.q_rx[i])) + continue; + mt76_queue_reset(dev, &dev->mt76.q_rx[i]); } diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/eeprom.c b/drivers/net/wireless/mediatek/mt76/mt7996/eeprom.c index 544b6c6f1ea329..4a823711828735 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/eeprom.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/eeprom.c @@ -14,7 +14,9 @@ static int mt7996_check_eeprom(struct mt7996_dev *dev) switch (val) { case 0x7990: - return 0; + return is_mt7996(&dev->mt76) ? 0 : -EINVAL; + case 0x7992: + return is_mt7992(&dev->mt76) ? 0 : -EINVAL; default: return -EINVAL; } @@ -22,8 +24,14 @@ static int mt7996_check_eeprom(struct mt7996_dev *dev) static char *mt7996_eeprom_name(struct mt7996_dev *dev) { - /* reserve for future variants */ - return MT7996_EEPROM_DEFAULT; + switch (mt76_chip(&dev->mt76)) { + case 0x7990: + return MT7996_EEPROM_DEFAULT; + case 0x7992: + return MT7992_EEPROM_DEFAULT; + default: + return MT7996_EEPROM_DEFAULT; + } } static int @@ -103,7 +111,8 @@ static int mt7996_eeprom_parse_efuse_hw_cap(struct mt7996_dev *dev) dev->wtbl_size_group = u32_get_bits(cap, WTBL_SIZE_GROUP); } - if (dev->wtbl_size_group < 2 || dev->wtbl_size_group > 4) + if (dev->wtbl_size_group < 2 || dev->wtbl_size_group > 4 || + is_mt7992(&dev->mt76)) dev->wtbl_size_group = 2; /* set default */ return 0; @@ -148,36 +157,49 @@ static int mt7996_eeprom_parse_band_config(struct mt7996_phy *phy) int mt7996_eeprom_parse_hw_cap(struct mt7996_dev *dev, struct mt7996_phy *phy) { - u8 path, nss, band_idx = phy->mt76->band_idx; + u8 path, rx_path, nss, band_idx = phy->mt76->band_idx; u8 *eeprom = dev->mt76.eeprom.data; struct mt76_phy *mphy = phy->mt76; + int max_path = 5, max_nss = 4; int ret; switch (band_idx) { case MT_BAND1: path = FIELD_GET(MT_EE_WIFI_CONF2_TX_PATH_BAND1, eeprom[MT_EE_WIFI_CONF + 2]); + rx_path = FIELD_GET(MT_EE_WIFI_CONF3_RX_PATH_BAND1, + eeprom[MT_EE_WIFI_CONF + 3]); nss = FIELD_GET(MT_EE_WIFI_CONF5_STREAM_NUM_BAND1, eeprom[MT_EE_WIFI_CONF + 5]); break; case MT_BAND2: path = FIELD_GET(MT_EE_WIFI_CONF2_TX_PATH_BAND2, eeprom[MT_EE_WIFI_CONF + 2]); + rx_path = FIELD_GET(MT_EE_WIFI_CONF4_RX_PATH_BAND2, + eeprom[MT_EE_WIFI_CONF + 4]); nss = FIELD_GET(MT_EE_WIFI_CONF5_STREAM_NUM_BAND2, eeprom[MT_EE_WIFI_CONF + 5]); break; default: path = FIELD_GET(MT_EE_WIFI_CONF1_TX_PATH_BAND0, eeprom[MT_EE_WIFI_CONF + 1]); + rx_path = FIELD_GET(MT_EE_WIFI_CONF3_RX_PATH_BAND0, + eeprom[MT_EE_WIFI_CONF + 3]); nss = FIELD_GET(MT_EE_WIFI_CONF4_STREAM_NUM_BAND0, eeprom[MT_EE_WIFI_CONF + 4]); break; } - if (!path || path > 4) - path = 4; + if (!path || path > max_path) + path = max_path; + + if (!nss || nss > max_nss) + nss = max_nss; + + nss = min_t(u8, nss, path); - nss = min_t(u8, min_t(u8, 4, nss), path); + if (path != rx_path) + phy->has_aux_rx = true; mphy->antenna_mask = BIT(nss) - 1; mphy->chainmask = (BIT(path) - 1) << dev->chainshift[band_idx]; diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/eeprom.h b/drivers/net/wireless/mediatek/mt76/mt7996/eeprom.h index 0c749774f6b10f..412d6e2f8014c1 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/eeprom.h +++ b/drivers/net/wireless/mediatek/mt76/mt7996/eeprom.h @@ -33,6 +33,9 @@ enum mt7996_eeprom_field { #define MT_EE_WIFI_CONF1_TX_PATH_BAND0 GENMASK(5, 3) #define MT_EE_WIFI_CONF2_TX_PATH_BAND1 GENMASK(2, 0) #define MT_EE_WIFI_CONF2_TX_PATH_BAND2 GENMASK(5, 3) +#define MT_EE_WIFI_CONF3_RX_PATH_BAND0 GENMASK(2, 0) +#define MT_EE_WIFI_CONF3_RX_PATH_BAND1 GENMASK(5, 3) +#define MT_EE_WIFI_CONF4_RX_PATH_BAND2 GENMASK(2, 0) #define MT_EE_WIFI_CONF4_STREAM_NUM_BAND0 GENMASK(5, 3) #define MT_EE_WIFI_CONF5_STREAM_NUM_BAND1 GENMASK(2, 0) #define MT_EE_WIFI_CONF5_STREAM_NUM_BAND2 GENMASK(5, 3) diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/init.c b/drivers/net/wireless/mediatek/mt76/mt7996/init.c index 55cb1770fa34e1..0cf0d1fe420a2e 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/init.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/init.c @@ -5,6 +5,8 @@ #include #include +#include +#include #include #include "mt7996.h" #include "mac.h" @@ -43,6 +45,183 @@ static const struct ieee80211_iface_combination if_comb[] = { } }; +static ssize_t mt7996_thermal_temp_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct mt7996_phy *phy = dev_get_drvdata(dev); + int i = to_sensor_dev_attr(attr)->index; + int temperature; + + switch (i) { + case 0: + temperature = mt7996_mcu_get_temperature(phy); + if (temperature < 0) + return temperature; + /* display in millidegree celcius */ + return sprintf(buf, "%u\n", temperature * 1000); + case 1: + case 2: + return sprintf(buf, "%u\n", + phy->throttle_temp[i - 1] * 1000); + case 3: + return sprintf(buf, "%hhu\n", phy->throttle_state); + default: + return -EINVAL; + } +} + +static ssize_t mt7996_thermal_temp_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct mt7996_phy *phy = dev_get_drvdata(dev); + int ret, i = to_sensor_dev_attr(attr)->index; + long val; + + ret = kstrtol(buf, 10, &val); + if (ret < 0) + return ret; + + mutex_lock(&phy->dev->mt76.mutex); + val = clamp_val(DIV_ROUND_CLOSEST(val, 1000), 40, 130); + + /* add a safety margin ~10 */ + if ((i - 1 == MT7996_CRIT_TEMP_IDX && + val > phy->throttle_temp[MT7996_MAX_TEMP_IDX] - 10) || + (i - 1 == MT7996_MAX_TEMP_IDX && + val - 10 < phy->throttle_temp[MT7996_CRIT_TEMP_IDX])) { + dev_err(phy->dev->mt76.dev, + "temp1_max shall be 10 degrees higher than temp1_crit."); + mutex_unlock(&phy->dev->mt76.mutex); + return -EINVAL; + } + + phy->throttle_temp[i - 1] = val; + mutex_unlock(&phy->dev->mt76.mutex); + + ret = mt7996_mcu_set_thermal_protect(phy, true); + if (ret) + return ret; + + return count; +} + +static SENSOR_DEVICE_ATTR_RO(temp1_input, mt7996_thermal_temp, 0); +static SENSOR_DEVICE_ATTR_RW(temp1_crit, mt7996_thermal_temp, 1); +static SENSOR_DEVICE_ATTR_RW(temp1_max, mt7996_thermal_temp, 2); +static SENSOR_DEVICE_ATTR_RO(throttle1, mt7996_thermal_temp, 3); + +static struct attribute *mt7996_hwmon_attrs[] = { + &sensor_dev_attr_temp1_input.dev_attr.attr, + &sensor_dev_attr_temp1_crit.dev_attr.attr, + &sensor_dev_attr_temp1_max.dev_attr.attr, + &sensor_dev_attr_throttle1.dev_attr.attr, + NULL, +}; +ATTRIBUTE_GROUPS(mt7996_hwmon); + +static int +mt7996_thermal_get_max_throttle_state(struct thermal_cooling_device *cdev, + unsigned long *state) +{ + *state = MT7996_CDEV_THROTTLE_MAX; + + return 0; +} + +static int +mt7996_thermal_get_cur_throttle_state(struct thermal_cooling_device *cdev, + unsigned long *state) +{ + struct mt7996_phy *phy = cdev->devdata; + + *state = phy->cdev_state; + + return 0; +} + +static int +mt7996_thermal_set_cur_throttle_state(struct thermal_cooling_device *cdev, + unsigned long state) +{ + struct mt7996_phy *phy = cdev->devdata; + u8 throttling = MT7996_THERMAL_THROTTLE_MAX - state; + int ret; + + if (state > MT7996_CDEV_THROTTLE_MAX) { + dev_err(phy->dev->mt76.dev, + "please specify a valid throttling state\n"); + return -EINVAL; + } + + if (state == phy->cdev_state) + return 0; + + /* cooling_device convention: 0 = no cooling, more = more cooling + * mcu convention: 1 = max cooling, more = less cooling + */ + ret = mt7996_mcu_set_thermal_throttling(phy, throttling); + if (ret) + return ret; + + phy->cdev_state = state; + + return 0; +} + +static const struct thermal_cooling_device_ops mt7996_thermal_ops = { + .get_max_state = mt7996_thermal_get_max_throttle_state, + .get_cur_state = mt7996_thermal_get_cur_throttle_state, + .set_cur_state = mt7996_thermal_set_cur_throttle_state, +}; + +static void mt7996_unregister_thermal(struct mt7996_phy *phy) +{ + struct wiphy *wiphy = phy->mt76->hw->wiphy; + + if (!phy->cdev) + return; + + sysfs_remove_link(&wiphy->dev.kobj, "cooling_device"); + thermal_cooling_device_unregister(phy->cdev); +} + +static int mt7996_thermal_init(struct mt7996_phy *phy) +{ + struct wiphy *wiphy = phy->mt76->hw->wiphy; + struct thermal_cooling_device *cdev; + struct device *hwmon; + const char *name; + + name = devm_kasprintf(&wiphy->dev, GFP_KERNEL, "mt7996_%s", + wiphy_name(wiphy)); + + cdev = thermal_cooling_device_register(name, phy, &mt7996_thermal_ops); + if (!IS_ERR(cdev)) { + if (sysfs_create_link(&wiphy->dev.kobj, &cdev->device.kobj, + "cooling_device") < 0) + thermal_cooling_device_unregister(cdev); + else + phy->cdev = cdev; + } + + /* initialize critical/maximum high temperature */ + phy->throttle_temp[MT7996_CRIT_TEMP_IDX] = MT7996_CRIT_TEMP; + phy->throttle_temp[MT7996_MAX_TEMP_IDX] = MT7996_MAX_TEMP; + + if (!IS_REACHABLE(CONFIG_HWMON)) + return 0; + + hwmon = devm_hwmon_device_register_with_groups(&wiphy->dev, name, phy, + mt7996_hwmon_groups); + + if (IS_ERR(hwmon)) + return PTR_ERR(hwmon); + + return 0; +} + static void mt7996_led_set_config(struct led_classdev *led_cdev, u8 delay_on, u8 delay_off) { @@ -109,10 +288,11 @@ static void mt7996_led_set_brightness(struct led_classdev *led_cdev, mt7996_led_set_config(led_cdev, 0xff, 0); } -void mt7996_init_txpower(struct mt7996_dev *dev, - struct ieee80211_supported_band *sband) +static void __mt7996_init_txpower(struct mt7996_phy *phy, + struct ieee80211_supported_band *sband) { - int i, nss = hweight8(dev->mphy.antenna_mask); + struct mt7996_dev *dev = phy->dev; + int i, nss = hweight16(phy->mt76->chainmask); int nss_delta = mt76_tx_power_nss_delta(nss); int pwr_delta = mt7996_eeprom_get_power_delta(dev, sband->band); struct mt76_power_limits limits; @@ -122,7 +302,7 @@ void mt7996_init_txpower(struct mt7996_dev *dev, int target_power = mt7996_eeprom_get_target_power(dev, chan); target_power += pwr_delta; - target_power = mt76_get_rate_power_limits(&dev->mphy, chan, + target_power = mt76_get_rate_power_limits(phy->mt76, chan, &limits, target_power); target_power += nss_delta; @@ -133,6 +313,19 @@ void mt7996_init_txpower(struct mt7996_dev *dev, } } +void mt7996_init_txpower(struct mt7996_phy *phy) +{ + if (!phy) + return; + + if (phy->mt76->cap.has_2ghz) + __mt7996_init_txpower(phy, &phy->mt76->sband_2g.sband); + if (phy->mt76->cap.has_5ghz) + __mt7996_init_txpower(phy, &phy->mt76->sband_5g.sband); + if (phy->mt76->cap.has_6ghz) + __mt7996_init_txpower(phy, &phy->mt76->sband_6g.sband); +} + static void mt7996_regd_notifier(struct wiphy *wiphy, struct regulatory_request *request) @@ -147,16 +340,14 @@ mt7996_regd_notifier(struct wiphy *wiphy, if (dev->mt76.region == NL80211_DFS_UNSET) mt7996_mcu_rdd_background_enable(phy, NULL); - mt7996_init_txpower(dev, &phy->mt76->sband_2g.sband); - mt7996_init_txpower(dev, &phy->mt76->sband_5g.sband); - mt7996_init_txpower(dev, &phy->mt76->sband_6g.sband); + mt7996_init_txpower(phy); phy->mt76->dfs_state = MT_DFS_STATE_UNKNOWN; mt7996_dfs_init_radar_detector(phy); } static void -mt7996_init_wiphy(struct ieee80211_hw *hw) +mt7996_init_wiphy(struct ieee80211_hw *hw, struct mtk_wed_device *wed) { struct mt7996_phy *phy = mt7996_hw_phy(hw); struct mt76_dev *mdev = &phy->dev->mt76; @@ -168,11 +359,14 @@ mt7996_init_wiphy(struct ieee80211_hw *hw) hw->max_rx_aggregation_subframes = max_subframes; hw->max_tx_aggregation_subframes = max_subframes; hw->netdev_features = NETIF_F_RXCSUM; + if (mtk_wed_device_active(wed)) + hw->netdev_features |= NETIF_F_HW_TC; hw->radiotap_timestamp.units_pos = IEEE80211_RADIOTAP_TIMESTAMP_UNIT_US; phy->slottime = 9; + phy->beacon_rate = -1; hw->sta_data_size = sizeof(struct mt7996_sta); hw->vif_data_size = sizeof(struct mt7996_vif); @@ -242,6 +436,7 @@ mt7996_init_wiphy(struct ieee80211_hw *hw) mt76_set_stream_caps(phy->mt76, true); mt7996_set_stream_vht_txbf_caps(phy); mt7996_set_stream_he_eht_caps(phy); + mt7996_init_txpower(phy); wiphy->available_antennas_rx = phy->mt76->antenna_mask; wiphy->available_antennas_tx = phy->mt76->antenna_mask; @@ -287,11 +482,12 @@ static void mt7996_mac_init_basic_rates(struct mt7996_dev *dev) for (i = 0; i < ARRAY_SIZE(mt76_rates); i++) { u16 rate = mt76_rates[i].hw_value; - u16 idx = MT7996_BASIC_RATES_TBL + i; + /* odd index for driver, even index for firmware */ + u16 idx = MT7996_BASIC_RATES_TBL + 2 * i; rate = FIELD_PREP(MT_TX_RATE_MODE, rate >> 8) | FIELD_PREP(MT_TX_RATE_IDX, rate & GENMASK(7, 0)); - mt7996_mac_set_fixed_rate_table(dev, idx, rate); + mt7996_mcu_set_fixed_rate_table(&dev->phy, idx, rate, false); } } @@ -317,9 +513,23 @@ void mt7996_mac_init(struct mt7996_dev *dev) mt76_rmw_field(dev, MT_DMA_TCRF1(2), MT_DMA_TCRF1_QIDX, 0); /* rro module init */ - mt7996_mcu_set_rro(dev, UNI_RRO_SET_PLATFORM_TYPE, 2); - mt7996_mcu_set_rro(dev, UNI_RRO_SET_BYPASS_MODE, 3); - mt7996_mcu_set_rro(dev, UNI_RRO_SET_TXFREE_PATH, 1); + if (is_mt7996(&dev->mt76)) + mt7996_mcu_set_rro(dev, UNI_RRO_SET_PLATFORM_TYPE, 2); + else + mt7996_mcu_set_rro(dev, UNI_RRO_SET_PLATFORM_TYPE, + dev->hif2 ? 7 : 0); + + if (dev->has_rro) { + u16 timeout; + + timeout = mt76_rr(dev, MT_HW_REV) == MT_HW_REV1 ? 512 : 128; + mt7996_mcu_set_rro(dev, UNI_RRO_SET_FLUSH_TIMEOUT, timeout); + mt7996_mcu_set_rro(dev, UNI_RRO_SET_BYPASS_MODE, 1); + mt7996_mcu_set_rro(dev, UNI_RRO_SET_TXFREE_PATH, 0); + } else { + mt7996_mcu_set_rro(dev, UNI_RRO_SET_BYPASS_MODE, 3); + mt7996_mcu_set_rro(dev, UNI_RRO_SET_TXFREE_PATH, 1); + } mt7996_mcu_wa_cmd(dev, MCU_WA_PARAM_CMD(SET), MCU_WA_PARAM_HW_PATH_HIF_VER, @@ -335,7 +545,8 @@ int mt7996_txbf_init(struct mt7996_dev *dev) { int ret; - if (dev->dbdc_support) { + if (mt7996_band_valid(dev, MT_BAND1) || + mt7996_band_valid(dev, MT_BAND2)) { ret = mt7996_mcu_set_txbf(dev, BF_MOD_EN_CTRL); if (ret) return ret; @@ -356,19 +567,18 @@ static int mt7996_register_phy(struct mt7996_dev *dev, struct mt7996_phy *phy, struct mt76_phy *mphy; u32 mac_ofs, hif1_ofs = 0; int ret; + struct mtk_wed_device *wed = &dev->mt76.mmio.wed; - if (band != MT_BAND1 && band != MT_BAND2) - return 0; - - if ((band == MT_BAND1 && !dev->dbdc_support) || - (band == MT_BAND2 && !dev->tbtc_support)) + if (!mt7996_band_valid(dev, band) || band == MT_BAND0) return 0; if (phy) return 0; - if (band == MT_BAND2 && dev->hif2) + if (is_mt7996(&dev->mt76) && band == MT_BAND2 && dev->hif2) { hif1_ofs = MT_WFDMA0_PCIE1(0) - MT_WFDMA0(0); + wed = &dev->mt76.mmio.wed_hif2; + } mphy = mt76_alloc_phy(&dev->mt76, sizeof(*phy), &mt7996_ops, band); if (!mphy) @@ -401,11 +611,12 @@ static int mt7996_register_phy(struct mt7996_dev *dev, struct mt7996_phy *phy, mt76_eeprom_override(mphy); /* init wiphy according to mphy and phy */ - mt7996_init_wiphy(mphy->hw); - ret = mt76_connac_init_tx_queues(phy->mt76, - MT_TXQ_ID(band), - MT7996_TX_RING_SIZE, - MT_TXQ_RING_BASE(band) + hif1_ofs, 0); + mt7996_init_wiphy(mphy->hw, wed); + ret = mt7996_init_tx_queues(mphy->priv, + MT_TXQ_ID(band), + MT7996_TX_RING_SIZE, + MT_TXQ_RING_BASE(band) + hif1_ofs, + wed); if (ret) goto error; @@ -414,10 +625,21 @@ static int mt7996_register_phy(struct mt7996_dev *dev, struct mt7996_phy *phy, if (ret) goto error; + ret = mt7996_thermal_init(phy); + if (ret) + goto error; + ret = mt7996_init_debugfs(phy); if (ret) goto error; + if (wed == &dev->mt76.mmio.wed_hif2 && mtk_wed_device_active(wed)) { + u32 irq_mask = dev->mt76.mmio.irqmask | MT_INT_TX_DONE_BAND2; + + mt76_wr(dev, MT_INT1_MASK_CSR, irq_mask); + mtk_wed_device_start(&dev->mt76.mmio.wed_hif2, irq_mask); + } + return 0; error: @@ -434,6 +656,8 @@ mt7996_unregister_phy(struct mt7996_phy *phy, enum mt76_band_id band) if (!phy) return; + mt7996_unregister_thermal(phy); + mphy = phy->dev->mt76.phys[band]; mt76_unregister_phy(mphy); ieee80211_free_hw(mphy->hw); @@ -447,9 +671,6 @@ static void mt7996_init_work(struct work_struct *work) mt7996_mcu_set_eeprom(dev); mt7996_mac_init(dev); - mt7996_init_txpower(dev, &dev->mphy.sband_2g.sband); - mt7996_init_txpower(dev, &dev->mphy.sband_5g.sband); - mt7996_init_txpower(dev, &dev->mphy.sband_6g.sband); mt7996_txbf_init(dev); } @@ -462,16 +683,225 @@ void mt7996_wfsys_reset(struct mt7996_dev *dev) msleep(20); } +static int mt7996_wed_rro_init(struct mt7996_dev *dev) +{ +#ifdef CONFIG_NET_MEDIATEK_SOC_WED + struct mtk_wed_device *wed = &dev->mt76.mmio.wed; + u32 reg = MT_RRO_ADDR_ELEM_SEG_ADDR0; + struct mt7996_wed_rro_addr *addr; + void *ptr; + int i; + + if (!dev->has_rro) + return 0; + + if (!mtk_wed_device_active(wed)) + return 0; + + for (i = 0; i < ARRAY_SIZE(dev->wed_rro.ba_bitmap); i++) { + ptr = dmam_alloc_coherent(dev->mt76.dma_dev, + MT7996_RRO_BA_BITMAP_CR_SIZE, + &dev->wed_rro.ba_bitmap[i].phy_addr, + GFP_KERNEL); + if (!ptr) + return -ENOMEM; + + dev->wed_rro.ba_bitmap[i].ptr = ptr; + } + + for (i = 0; i < ARRAY_SIZE(dev->wed_rro.addr_elem); i++) { + int j; + + ptr = dmam_alloc_coherent(dev->mt76.dma_dev, + MT7996_RRO_WINDOW_MAX_SIZE * sizeof(*addr), + &dev->wed_rro.addr_elem[i].phy_addr, + GFP_KERNEL); + if (!ptr) + return -ENOMEM; + + dev->wed_rro.addr_elem[i].ptr = ptr; + memset(dev->wed_rro.addr_elem[i].ptr, 0, + MT7996_RRO_WINDOW_MAX_SIZE * sizeof(*addr)); + + addr = dev->wed_rro.addr_elem[i].ptr; + for (j = 0; j < MT7996_RRO_WINDOW_MAX_SIZE; j++) { + addr->signature = 0xff; + addr++; + } + + wed->wlan.ind_cmd.addr_elem_phys[i] = + dev->wed_rro.addr_elem[i].phy_addr; + } + + ptr = dmam_alloc_coherent(dev->mt76.dma_dev, + MT7996_RRO_WINDOW_MAX_LEN * sizeof(*addr), + &dev->wed_rro.session.phy_addr, + GFP_KERNEL); + if (!ptr) + return -ENOMEM; + + dev->wed_rro.session.ptr = ptr; + addr = dev->wed_rro.session.ptr; + for (i = 0; i < MT7996_RRO_WINDOW_MAX_LEN; i++) { + addr->signature = 0xff; + addr++; + } + + /* rro hw init */ + /* TODO: remove line after WM has set */ + mt76_clear(dev, WF_RRO_AXI_MST_CFG, WF_RRO_AXI_MST_CFG_DIDX_OK); + + /* setup BA bitmap cache address */ + mt76_wr(dev, MT_RRO_BA_BITMAP_BASE0, + dev->wed_rro.ba_bitmap[0].phy_addr); + mt76_wr(dev, MT_RRO_BA_BITMAP_BASE1, 0); + mt76_wr(dev, MT_RRO_BA_BITMAP_BASE_EXT0, + dev->wed_rro.ba_bitmap[1].phy_addr); + mt76_wr(dev, MT_RRO_BA_BITMAP_BASE_EXT1, 0); + + /* setup Address element address */ + for (i = 0; i < ARRAY_SIZE(dev->wed_rro.addr_elem); i++) { + mt76_wr(dev, reg, dev->wed_rro.addr_elem[i].phy_addr >> 4); + reg += 4; + } + + /* setup Address element address - separate address segment mode */ + mt76_wr(dev, MT_RRO_ADDR_ARRAY_BASE1, + MT_RRO_ADDR_ARRAY_ELEM_ADDR_SEG_MODE); + + wed->wlan.ind_cmd.win_size = ffs(MT7996_RRO_WINDOW_MAX_LEN) - 6; + wed->wlan.ind_cmd.particular_sid = MT7996_RRO_MAX_SESSION; + wed->wlan.ind_cmd.particular_se_phys = dev->wed_rro.session.phy_addr; + wed->wlan.ind_cmd.se_group_nums = MT7996_RRO_ADDR_ELEM_LEN; + wed->wlan.ind_cmd.ack_sn_addr = MT_RRO_ACK_SN_CTRL; + + mt76_wr(dev, MT_RRO_IND_CMD_SIGNATURE_BASE0, 0x15010e00); + mt76_set(dev, MT_RRO_IND_CMD_SIGNATURE_BASE1, + MT_RRO_IND_CMD_SIGNATURE_BASE1_EN); + + /* particular session configure */ + /* use max session idx + 1 as particular session id */ + mt76_wr(dev, MT_RRO_PARTICULAR_CFG0, dev->wed_rro.session.phy_addr); + mt76_wr(dev, MT_RRO_PARTICULAR_CFG1, + MT_RRO_PARTICULAR_CONFG_EN | + FIELD_PREP(MT_RRO_PARTICULAR_SID, MT7996_RRO_MAX_SESSION)); + + /* interrupt enable */ + mt76_wr(dev, MT_RRO_HOST_INT_ENA, + MT_RRO_HOST_INT_ENA_HOST_RRO_DONE_ENA); + + /* rro ind cmd queue init */ + return mt7996_dma_rro_init(dev); +#else + return 0; +#endif +} + +static void mt7996_wed_rro_free(struct mt7996_dev *dev) +{ +#ifdef CONFIG_NET_MEDIATEK_SOC_WED + int i; + + if (!dev->has_rro) + return; + + if (!mtk_wed_device_active(&dev->mt76.mmio.wed)) + return; + + for (i = 0; i < ARRAY_SIZE(dev->wed_rro.ba_bitmap); i++) { + if (!dev->wed_rro.ba_bitmap[i].ptr) + continue; + + dmam_free_coherent(dev->mt76.dma_dev, + MT7996_RRO_BA_BITMAP_CR_SIZE, + dev->wed_rro.ba_bitmap[i].ptr, + dev->wed_rro.ba_bitmap[i].phy_addr); + } + + for (i = 0; i < ARRAY_SIZE(dev->wed_rro.addr_elem); i++) { + if (!dev->wed_rro.addr_elem[i].ptr) + continue; + + dmam_free_coherent(dev->mt76.dma_dev, + MT7996_RRO_WINDOW_MAX_SIZE * + sizeof(struct mt7996_wed_rro_addr), + dev->wed_rro.addr_elem[i].ptr, + dev->wed_rro.addr_elem[i].phy_addr); + } + + if (!dev->wed_rro.session.ptr) + return; + + dmam_free_coherent(dev->mt76.dma_dev, + MT7996_RRO_WINDOW_MAX_LEN * + sizeof(struct mt7996_wed_rro_addr), + dev->wed_rro.session.ptr, + dev->wed_rro.session.phy_addr); +#endif +} + +static void mt7996_wed_rro_work(struct work_struct *work) +{ +#ifdef CONFIG_NET_MEDIATEK_SOC_WED + struct mt7996_dev *dev; + LIST_HEAD(list); + + dev = (struct mt7996_dev *)container_of(work, struct mt7996_dev, + wed_rro.work); + + spin_lock_bh(&dev->wed_rro.lock); + list_splice_init(&dev->wed_rro.poll_list, &list); + spin_unlock_bh(&dev->wed_rro.lock); + + while (!list_empty(&list)) { + struct mt7996_wed_rro_session_id *e; + int i; + + e = list_first_entry(&list, struct mt7996_wed_rro_session_id, + list); + list_del_init(&e->list); + + for (i = 0; i < MT7996_RRO_WINDOW_MAX_LEN; i++) { + void *ptr = dev->wed_rro.session.ptr; + struct mt7996_wed_rro_addr *elem; + u32 idx, elem_id = i; + + if (e->id == MT7996_RRO_MAX_SESSION) + goto reset; + + idx = e->id / MT7996_RRO_BA_BITMAP_SESSION_SIZE; + if (idx >= ARRAY_SIZE(dev->wed_rro.addr_elem)) + goto out; + + ptr = dev->wed_rro.addr_elem[idx].ptr; + elem_id += + (e->id % MT7996_RRO_BA_BITMAP_SESSION_SIZE) * + MT7996_RRO_WINDOW_MAX_LEN; +reset: + elem = ptr + elem_id * sizeof(*elem); + elem->signature = 0xff; + } + mt7996_mcu_wed_rro_reset_sessions(dev, e->id); +out: + kfree(e); + } +#endif +} + static int mt7996_init_hardware(struct mt7996_dev *dev) { int ret, idx; mt76_wr(dev, MT_INT_SOURCE_CSR, ~0); + if (is_mt7992(&dev->mt76)) { + mt76_rmw(dev, MT_AFE_CTL_BAND_PLL_03(MT_BAND0), MT_AFE_CTL_BAND_PLL_03_MSB_EN, 0); + mt76_rmw(dev, MT_AFE_CTL_BAND_PLL_03(MT_BAND1), MT_AFE_CTL_BAND_PLL_03_MSB_EN, 0); + } INIT_WORK(&dev->init_work, mt7996_init_work); - - dev->dbdc_support = true; - dev->tbtc_support = true; + INIT_WORK(&dev->wed_rro.work, mt7996_wed_rro_work); + INIT_LIST_HEAD(&dev->wed_rro.poll_list); + spin_lock_init(&dev->wed_rro.lock); ret = mt7996_dma_init(dev); if (ret) @@ -483,6 +913,10 @@ static int mt7996_init_hardware(struct mt7996_dev *dev) if (ret) return ret; + ret = mt7996_wed_rro_init(dev); + if (ret) + return ret; + ret = mt7996_eeprom_init(dev); if (ret < 0) return ret; @@ -889,14 +1323,16 @@ int mt7996_register_device(struct mt7996_dev *dev) if (ret) return ret; - mt7996_init_wiphy(hw); + mt7996_init_wiphy(hw, &dev->mt76.mmio.wed); ret = mt76_register_device(&dev->mt76, true, mt76_rates, ARRAY_SIZE(mt76_rates)); if (ret) return ret; - ieee80211_queue_work(mt76_hw(dev), &dev->init_work); + ret = mt7996_thermal_init(&dev->phy); + if (ret) + return ret; ret = mt7996_register_phy(dev, mt7996_phy2(dev), MT_BAND1); if (ret) @@ -906,21 +1342,35 @@ int mt7996_register_device(struct mt7996_dev *dev) if (ret) return ret; + ieee80211_queue_work(mt76_hw(dev), &dev->init_work); + dev->recovery.hw_init_done = true; ret = mt7996_init_debugfs(&dev->phy); if (ret) - return ret; + goto error; + + ret = mt7996_coredump_register(dev); + if (ret) + goto error; - return mt7996_coredump_register(dev); + return 0; + +error: + cancel_work_sync(&dev->init_work); + + return ret; } void mt7996_unregister_device(struct mt7996_dev *dev) { + cancel_work_sync(&dev->wed_rro.work); mt7996_unregister_phy(mt7996_phy3(dev), MT_BAND2); mt7996_unregister_phy(mt7996_phy2(dev), MT_BAND1); + mt7996_unregister_thermal(&dev->phy); mt7996_coredump_unregister(dev); mt76_unregister_device(&dev->mt76); + mt7996_wed_rro_free(dev); mt7996_mcu_exit(dev); mt7996_tx_token_put(dev); mt7996_dma_cleanup(dev); diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mac.c b/drivers/net/wireless/mediatek/mt76/mt7996/mac.c index 04540833485fe8..53258488d49f36 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mac.c @@ -102,7 +102,6 @@ static void mt7996_mac_sta_poll(struct mt7996_dev *dev) }; struct ieee80211_sta *sta; struct mt7996_sta *msta; - struct rate_info *rate; u32 tx_time[IEEE80211_NUM_ACS], rx_time[IEEE80211_NUM_ACS]; LIST_HEAD(sta_poll_list); int i; @@ -118,7 +117,6 @@ static void mt7996_mac_sta_poll(struct mt7996_dev *dev) u32 addr, val; u16 idx; s8 rssi[4]; - u8 bw; spin_lock_bh(&dev->mt76.sta_poll_lock); if (list_empty(&sta_poll_list)) { @@ -174,49 +172,6 @@ static void mt7996_mac_sta_poll(struct mt7996_dev *dev) ieee80211_sta_register_airtime(sta, tid, tx_cur, rx_cur); } - /* We don't support reading GI info from txs packets. - * For accurate tx status reporting and AQL improvement, - * we need to make sure that flags match so polling GI - * from per-sta counters directly. - */ - rate = &msta->wcid.rate; - - switch (rate->bw) { - case RATE_INFO_BW_320: - bw = IEEE80211_STA_RX_BW_320; - break; - case RATE_INFO_BW_160: - bw = IEEE80211_STA_RX_BW_160; - break; - case RATE_INFO_BW_80: - bw = IEEE80211_STA_RX_BW_80; - break; - case RATE_INFO_BW_40: - bw = IEEE80211_STA_RX_BW_40; - break; - default: - bw = IEEE80211_STA_RX_BW_20; - break; - } - - addr = mt7996_mac_wtbl_lmac_addr(dev, idx, 6); - val = mt76_rr(dev, addr); - if (rate->flags & RATE_INFO_FLAGS_EHT_MCS) { - addr = mt7996_mac_wtbl_lmac_addr(dev, idx, 5); - val = mt76_rr(dev, addr); - rate->eht_gi = FIELD_GET(GENMASK(25, 24), val); - } else if (rate->flags & RATE_INFO_FLAGS_HE_MCS) { - u8 offs = 24 + 2 * bw; - - rate->he_gi = (val & (0x3 << offs)) >> offs; - } else if (rate->flags & - (RATE_INFO_FLAGS_VHT_MCS | RATE_INFO_FLAGS_MCS)) { - if (val & BIT(12 + bw)) - rate->flags |= RATE_INFO_FLAGS_SHORT_GI; - else - rate->flags &= ~RATE_INFO_FLAGS_SHORT_GI; - } - /* get signal strength of resp frames (CTS/BA/ACK) */ addr = mt7996_mac_wtbl_lmac_addr(dev, idx, 34); val = mt76_rr(dev, addr); @@ -248,17 +203,6 @@ void mt7996_mac_enable_rtscts(struct mt7996_dev *dev, mt76_clear(dev, addr, BIT(5)); } -void mt7996_mac_set_fixed_rate_table(struct mt7996_dev *dev, - u8 tbl_idx, u16 rate_idx) -{ - u32 ctrl = MT_WTBL_ITCR_WR | MT_WTBL_ITCR_EXEC | tbl_idx; - - mt76_wr(dev, MT_WTBL_ITDR0, rate_idx); - /* use wtbl spe idx */ - mt76_wr(dev, MT_WTBL_ITDR1, MT_WTBL_SPE_IDX_SEL); - mt76_wr(dev, MT_WTBL_ITCR, ctrl); -} - /* The HW does not translate the mac header to 802.3 for mesh point */ static int mt7996_reverse_frag0_hdr_trans(struct sk_buff *skb, u16 hdr_gap) { @@ -449,8 +393,36 @@ mt7996_mac_fill_rx_rate(struct mt7996_dev *dev, return 0; } +static void +mt7996_wed_check_ppe(struct mt7996_dev *dev, struct mt76_queue *q, + struct mt7996_sta *msta, struct sk_buff *skb, + u32 info) +{ + struct ieee80211_vif *vif; + struct wireless_dev *wdev; + + if (!msta || !msta->vif) + return; + + if (!mt76_queue_is_wed_rx(q)) + return; + + if (!(info & MT_DMA_INFO_PPE_VLD)) + return; + + vif = container_of((void *)msta->vif, struct ieee80211_vif, + drv_priv); + wdev = ieee80211_vif_to_wdev(vif); + skb->dev = wdev->netdev; + + mtk_wed_device_ppe_check(&dev->mt76.mmio.wed, skb, + FIELD_GET(MT_DMA_PPE_CPU_REASON, info), + FIELD_GET(MT_DMA_PPE_ENTRY, info)); +} + static int -mt7996_mac_fill_rx(struct mt7996_dev *dev, struct sk_buff *skb) +mt7996_mac_fill_rx(struct mt7996_dev *dev, enum mt76_rxq_id q, + struct sk_buff *skb, u32 *info) { struct mt76_rx_status *status = (struct mt76_rx_status *)skb->cb; struct mt76_phy *mphy = &dev->mt76.phy; @@ -475,7 +447,10 @@ mt7996_mac_fill_rx(struct mt7996_dev *dev, struct sk_buff *skb) u16 seq_ctrl = 0; __le16 fc = 0; int idx; + u8 hw_aggr = false; + struct mt7996_sta *msta = NULL; + hw_aggr = status->aggr; memset(status, 0, sizeof(*status)); band_idx = FIELD_GET(MT_RXD1_NORMAL_BAND_IDX, rxd1); @@ -502,8 +477,6 @@ mt7996_mac_fill_rx(struct mt7996_dev *dev, struct sk_buff *skb) status->wcid = mt7996_rx_get_wcid(dev, idx, unicast); if (status->wcid) { - struct mt7996_sta *msta; - msta = container_of(status->wcid, struct mt7996_sta, wcid); spin_lock_bh(&dev->mt76.sta_poll_lock); if (list_empty(&msta->wcid.poll_list)) @@ -708,12 +681,14 @@ mt7996_mac_fill_rx(struct mt7996_dev *dev, struct sk_buff *skb) } } else { status->flag |= RX_FLAG_8023; + mt7996_wed_check_ppe(dev, &dev->mt76.q_rx[q], msta, skb, + *info); } if (rxv && mode >= MT_PHY_TYPE_HE_SU && !(status->flag & RX_FLAG_8023)) mt76_connac3_mac_decode_he_radiotap(skb, rxv, mode); - if (!status->wcid || !ieee80211_is_data_qos(fc)) + if (!status->wcid || !ieee80211_is_data_qos(fc) || hw_aggr) return 0; status->aggr = unicast && @@ -840,10 +815,10 @@ void mt7996_mac_write_txwi(struct mt7996_dev *dev, __le32 *txwi, struct mt76_vif *mvif; u16 tx_count = 15; u32 val; - bool beacon = !!(changed & (BSS_CHANGED_BEACON | - BSS_CHANGED_BEACON_ENABLED)); bool inband_disc = !!(changed & (BSS_CHANGED_UNSOL_BCAST_PROBE_RESP | BSS_CHANGED_FILS_DISCOVERY)); + bool beacon = !!(changed & (BSS_CHANGED_BEACON | + BSS_CHANGED_BEACON_ENABLED)) && (!inband_disc); mvif = vif ? (struct mt76_vif *)vif->drv_priv : NULL; if (mvif) { @@ -898,8 +873,11 @@ void mt7996_mac_write_txwi(struct mt7996_dev *dev, __le32 *txwi, val |= MT_TXD5_TX_STATUS_HOST; txwi[5] = cpu_to_le32(val); - val = MT_TXD6_DIS_MAT | MT_TXD6_DAS | - FIELD_PREP(MT_TXD6_MSDU_CNT, 1); + val = MT_TXD6_DIS_MAT | MT_TXD6_DAS; + if (is_mt7996(&dev->mt76)) + val |= FIELD_PREP(MT_TXD6_MSDU_CNT, 1); + else + val |= FIELD_PREP(MT_TXD6_MSDU_CNT_V2, 1); txwi[6] = cpu_to_le32(val); txwi[7] = 0; @@ -923,7 +901,8 @@ void mt7996_mac_write_txwi(struct mt7996_dev *dev, __le32 *txwi, idx = mvif->basic_rates_idx; } - txwi[6] |= cpu_to_le32(FIELD_PREP(MT_TXD6_TX_RATE, idx)); + val = FIELD_PREP(MT_TXD6_TX_RATE, idx) | MT_TXD6_FIXED_BW; + txwi[6] |= cpu_to_le32(val); txwi[3] |= cpu_to_le32(MT_TXD3_BA_DISABLE); } } @@ -963,8 +942,16 @@ int mt7996_tx_prepare_skb(struct mt76_dev *mdev, void *txwi_ptr, txp = (struct mt76_connac_txp_common *)(txwi + MT_TXD_SIZE); for (i = 0; i < nbuf; i++) { + u16 len; + + len = FIELD_PREP(MT_TXP_BUF_LEN, tx_info->buf[i + 1].len); +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT + len |= FIELD_PREP(MT_TXP_DMA_ADDR_H, + tx_info->buf[i + 1].addr >> 32); +#endif + txp->fw.buf[i] = cpu_to_le32(tx_info->buf[i + 1].addr); - txp->fw.len[i] = cpu_to_le16(tx_info->buf[i + 1].len); + txp->fw.len[i] = cpu_to_le16(len); } txp->fw.nbuf = nbuf; @@ -996,6 +983,29 @@ int mt7996_tx_prepare_skb(struct mt76_dev *mdev, void *txwi_ptr, return 0; } +u32 mt7996_wed_init_buf(void *ptr, dma_addr_t phys, int token_id) +{ + struct mt76_connac_fw_txp *txp = ptr + MT_TXD_SIZE; + __le32 *txwi = ptr; + u32 val; + + memset(ptr, 0, MT_TXD_SIZE + sizeof(*txp)); + + val = FIELD_PREP(MT_TXD0_TX_BYTES, MT_TXD_SIZE) | + FIELD_PREP(MT_TXD0_PKT_FMT, MT_TX_TYPE_CT); + txwi[0] = cpu_to_le32(val); + + val = BIT(31) | + FIELD_PREP(MT_TXD1_HDR_FORMAT, MT_HDR_FORMAT_802_3); + txwi[1] = cpu_to_le32(val); + + txp->token = cpu_to_le16(token_id); + txp->nbuf = 1; + txp->buf[0] = cpu_to_le32(phys + MT_TXD_SIZE + sizeof(*txp)); + + return MT_TXD_SIZE + sizeof(*txp); +} + static void mt7996_tx_check_aggr(struct ieee80211_sta *sta, struct sk_buff *skb) { @@ -1074,7 +1084,7 @@ mt7996_mac_tx_free(struct mt7996_dev *dev, void *data, int len) struct mt76_phy *phy3 = mdev->phys[MT_BAND2]; struct mt76_txwi_cache *txwi; struct ieee80211_sta *sta = NULL; - struct mt76_wcid *wcid; + struct mt76_wcid *wcid = NULL; LIST_HEAD(free_list); struct sk_buff *skb, *tmp; void *end = data + len; @@ -1254,6 +1264,8 @@ mt7996_mac_add_txs_skb(struct mt7996_dev *dev, struct mt76_wcid *wcid, goto out; rate.flags = RATE_INFO_FLAGS_VHT_MCS; + if (wcid->rate.flags & RATE_INFO_FLAGS_SHORT_GI) + rate.flags |= RATE_INFO_FLAGS_SHORT_GI; break; case MT_PHY_TYPE_HE_SU: case MT_PHY_TYPE_HE_EXT_SU: @@ -1403,6 +1415,12 @@ void mt7996_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q, switch (type) { case PKT_TYPE_TXRX_NOTIFY: + if (mtk_wed_device_active(&dev->mt76.mmio.wed_hif2) && + q == MT_RXQ_TXFREE_BAND2) { + dev_kfree_skb(skb); + break; + } + mt7996_mac_tx_free(dev, skb->data, skb->len); napi_consume_skb(skb, 1); break; @@ -1419,7 +1437,7 @@ void mt7996_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q, dev_kfree_skb(skb); break; case PKT_TYPE_NORMAL: - if (!mt7996_mac_fill_rx(dev, skb)) { + if (!mt7996_mac_fill_rx(dev, q, skb, info)) { mt76_rx(&dev->mt76, q, skb); return; } @@ -1525,7 +1543,7 @@ mt7996_phy_get_nf(struct mt7996_phy *phy, u8 band_idx) void mt7996_update_channel(struct mt76_phy *mphy) { - struct mt7996_phy *phy = (struct mt7996_phy *)mphy->priv; + struct mt7996_phy *phy = mphy->priv; struct mt76_channel_state *state = mphy->chan_state; int nf; @@ -1652,6 +1670,10 @@ mt7996_mac_restart(struct mt7996_dev *dev) /* disable all tx/rx napi */ mt76_worker_disable(&dev->mt76.tx_worker); mt76_for_each_q_rx(mdev, i) { + if (mtk_wed_device_active(&dev->mt76.mmio.wed) && + mt76_queue_is_wed_rro(&mdev->q_rx[i])) + continue; + if (mdev->q_rx[i].ndesc) napi_disable(&dev->mt76.napi[i]); } @@ -1665,6 +1687,10 @@ mt7996_mac_restart(struct mt7996_dev *dev) local_bh_disable(); mt76_for_each_q_rx(mdev, i) { + if (mtk_wed_device_active(&dev->mt76.mmio.wed) && + mt76_queue_is_wed_rro(&mdev->q_rx[i])) + continue; + if (mdev->q_rx[i].ndesc) { napi_enable(&dev->mt76.napi[i]); napi_schedule(&dev->mt76.napi[i]); @@ -1697,9 +1723,9 @@ mt7996_mac_restart(struct mt7996_dev *dev) goto out; mt7996_mac_init(dev); - mt7996_init_txpower(dev, &dev->mphy.sband_2g.sband); - mt7996_init_txpower(dev, &dev->mphy.sband_5g.sband); - mt7996_init_txpower(dev, &dev->mphy.sband_6g.sband); + mt7996_init_txpower(&dev->phy); + mt7996_init_txpower(phy2); + mt7996_init_txpower(phy3); ret = mt7996_txbf_init(dev); if (test_bit(MT76_STATE_RUNNING, &dev->mphy.state)) { @@ -1754,6 +1780,7 @@ mt7996_mac_full_reset(struct mt7996_dev *dev) if (phy3) ieee80211_stop_queues(phy3->mt76->hw); + cancel_work_sync(&dev->wed_rro.work); cancel_delayed_work_sync(&dev->mphy.mac_work); if (phy2) cancel_delayed_work_sync(&phy2->mt76->mac_work); @@ -1836,6 +1863,13 @@ void mt7996_mac_reset_work(struct work_struct *work) dev_info(dev->mt76.dev,"\n%s L1 SER recovery start.", wiphy_name(dev->mt76.hw->wiphy)); + + if (mtk_wed_device_active(&dev->mt76.mmio.wed_hif2)) + mtk_wed_device_stop(&dev->mt76.mmio.wed_hif2); + + if (mtk_wed_device_active(&dev->mt76.mmio.wed)) + mtk_wed_device_stop(&dev->mt76.mmio.wed); + ieee80211_stop_queues(mt76_hw(dev)); if (phy2) ieee80211_stop_queues(phy2->mt76->hw); @@ -1845,6 +1879,8 @@ void mt7996_mac_reset_work(struct work_struct *work) set_bit(MT76_RESET, &dev->mphy.state); set_bit(MT76_MCU_RESET, &dev->mphy.state); wake_up(&dev->mt76.mcu.wait); + + cancel_work_sync(&dev->wed_rro.work); cancel_delayed_work_sync(&dev->mphy.mac_work); if (phy2) { set_bit(MT76_RESET, &phy2->mt76->state); @@ -1855,8 +1891,13 @@ void mt7996_mac_reset_work(struct work_struct *work) cancel_delayed_work_sync(&phy3->mt76->mac_work); } mt76_worker_disable(&dev->mt76.tx_worker); - mt76_for_each_q_rx(&dev->mt76, i) + mt76_for_each_q_rx(&dev->mt76, i) { + if (mtk_wed_device_active(&dev->mt76.mmio.wed) && + mt76_queue_is_wed_rro(&dev->mt76.q_rx[i])) + continue; + napi_disable(&dev->mt76.napi[i]); + } napi_disable(&dev->mt76.tx_napi); mutex_lock(&dev->mt76.mutex); @@ -1877,7 +1918,28 @@ void mt7996_mac_reset_work(struct work_struct *work) mt7996_wait_reset_state(dev, MT_MCU_CMD_NORMAL_STATE); /* enable DMA Tx/Tx and interrupt */ - mt7996_dma_start(dev, false); + mt7996_dma_start(dev, false, false); + + if (mtk_wed_device_active(&dev->mt76.mmio.wed)) { + u32 wed_irq_mask = MT_INT_RRO_RX_DONE | MT_INT_TX_DONE_BAND2 | + dev->mt76.mmio.irqmask; + + if (mtk_wed_get_rx_capa(&dev->mt76.mmio.wed)) + wed_irq_mask &= ~MT_INT_RX_DONE_RRO_IND; + + mt76_wr(dev, MT_INT_MASK_CSR, wed_irq_mask); + + mtk_wed_device_start_hw_rro(&dev->mt76.mmio.wed, wed_irq_mask, + true); + mt7996_irq_enable(dev, wed_irq_mask); + mt7996_irq_disable(dev, 0); + } + + if (mtk_wed_device_active(&dev->mt76.mmio.wed_hif2)) { + mt76_wr(dev, MT_INT_PCIE1_MASK_CSR, MT_INT_TX_RX_DONE_EXT); + mtk_wed_device_start(&dev->mt76.mmio.wed_hif2, + MT_INT_TX_RX_DONE_EXT); + } clear_bit(MT76_MCU_RESET, &dev->mphy.state); clear_bit(MT76_RESET, &dev->mphy.state); @@ -1888,6 +1950,10 @@ void mt7996_mac_reset_work(struct work_struct *work) local_bh_disable(); mt76_for_each_q_rx(&dev->mt76, i) { + if (mtk_wed_device_active(&dev->mt76.mmio.wed) && + mt76_queue_is_wed_rro(&dev->mt76.q_rx[i])) + continue; + napi_enable(&dev->mt76.napi[i]); napi_schedule(&dev->mt76.napi[i]); } @@ -2187,7 +2253,9 @@ void mt7996_mac_sta_rc_work(struct work_struct *work) IEEE80211_RC_BW_CHANGED)) mt7996_mcu_add_rate_ctrl(dev, vif, sta, true); - /* TODO: smps change */ + if (changed & IEEE80211_RC_SMPS_CHANGED) + mt7996_mcu_set_fixed_field(dev, vif, sta, NULL, + RATE_PARAM_MMPS_UPDATE); spin_lock_bh(&dev->mt76.sta_poll_lock); } @@ -2212,6 +2280,7 @@ void mt7996_mac_work(struct work_struct *work) mt7996_mac_update_stats(phy); + mt7996_mcu_get_all_sta_info(phy, UNI_ALL_STA_TXRX_RATE); if (mtk_wed_device_active(&phy->dev->mt76.mmio.wed)) { mt7996_mcu_get_all_sta_info(phy, UNI_ALL_STA_TXRX_ADM_STAT); mt7996_mcu_get_all_sta_info(phy, UNI_ALL_STA_TXRX_MSDU_COUNT); diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/main.c b/drivers/net/wireless/mediatek/mt76/mt7996/main.c index 09c7a28a3d511b..51deea84b64294 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/main.c @@ -51,6 +51,14 @@ int mt7996_run(struct ieee80211_hw *hw) if (ret) goto out; + ret = mt7996_mcu_set_thermal_throttling(phy, MT7996_THERMAL_THROTTLE_MAX); + if (ret) + goto out; + + ret = mt7996_mcu_set_thermal_protect(phy, true); + if (ret) + goto out; + set_bit(MT76_STATE_RUNNING, &phy->mt76->state); ieee80211_queue_delayed_work(hw, &phy->mt76->mac_work, @@ -342,6 +350,8 @@ static int mt7996_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd, case WLAN_CIPHER_SUITE_GCMP: case WLAN_CIPHER_SUITE_GCMP_256: case WLAN_CIPHER_SUITE_SMS4: + case WLAN_CIPHER_SUITE_BIP_GMAC_128: + case WLAN_CIPHER_SUITE_BIP_GMAC_256: break; case WLAN_CIPHER_SUITE_WEP40: case WLAN_CIPHER_SUITE_WEP104: @@ -365,9 +375,13 @@ static int mt7996_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd, } mt76_wcid_key_setup(&dev->mt76, wcid, key); - err = mt7996_mcu_add_key(&dev->mt76, vif, &msta->bip, - key, MCU_WMWA_UNI_CMD(STA_REC_UPDATE), - &msta->wcid, cmd); + + if (key->keyidx == 6 || key->keyidx == 7) + err = mt7996_mcu_bcn_prot_enable(dev, vif, key); + else + err = mt7996_mcu_add_key(&dev->mt76, vif, key, + MCU_WMWA_UNI_CMD(STA_REC_UPDATE), + &msta->wcid, cmd); out: mutex_unlock(&dev->mt76.mutex); @@ -388,6 +402,13 @@ static int mt7996_config(struct ieee80211_hw *hw, u32 changed) ieee80211_wake_queues(hw); } + if (changed & (IEEE80211_CONF_CHANGE_POWER | + IEEE80211_CONF_CHANGE_CHANNEL)) { + ret = mt7996_mcu_set_txpower_sku(phy); + if (ret) + return ret; + } + mutex_lock(&dev->mt76.mutex); if (changed & IEEE80211_CONF_CHANGE_MONITOR) { @@ -514,24 +535,25 @@ mt7996_get_rates_table(struct ieee80211_hw *hw, struct ieee80211_vif *vif, struct mt76_vif *mvif = (struct mt76_vif *)vif->drv_priv; struct mt76_phy *mphy = hw->priv; u16 rate; - u8 i, idx, ht; + u8 i, idx; rate = mt76_connac2_mac_tx_rate_val(mphy, vif, beacon, mcast); - ht = FIELD_GET(MT_TX_RATE_MODE, rate) > MT_PHY_TYPE_OFDM; - if (beacon && ht) { - struct mt7996_dev *dev = mt7996_hw_dev(hw); + if (beacon) { + struct mt7996_phy *phy = mphy->priv; + + /* odd index for driver, even index for firmware */ + idx = MT7996_BEACON_RATES_TBL + 2 * phy->mt76->band_idx; + if (phy->beacon_rate != rate) + mt7996_mcu_set_fixed_rate_table(phy, idx, rate, beacon); - /* must odd index */ - idx = MT7996_BEACON_RATES_TBL + 2 * (mvif->idx % 20); - mt7996_mac_set_fixed_rate_table(dev, idx, rate); return idx; } idx = FIELD_GET(MT_TX_RATE_IDX, rate); for (i = 0; i < ARRAY_SIZE(mt76_rates); i++) if ((mt76_rates[i].hw_value & GENMASK(7, 0)) == idx) - return MT7996_BASIC_RATES_TBL + i; + return MT7996_BASIC_RATES_TBL + 2 * i; return mvif->basic_rates_idx; } @@ -956,8 +978,8 @@ mt7996_set_antenna(struct ieee80211_hw *hw, u32 tx_ant, u32 rx_ant) mt76_set_stream_caps(phy->mt76, true); mt7996_set_stream_vht_txbf_caps(phy); mt7996_set_stream_he_eht_caps(phy); + mt7996_mcu_set_txpower_sku(phy); - /* TODO: update bmc_wtbl spe_idx when antenna changes */ mutex_unlock(&dev->mt76.mutex); return 0; @@ -982,6 +1004,7 @@ static void mt7996_sta_statistics(struct ieee80211_hw *hw, sinfo->txrate.he_gi = txrate->he_gi; sinfo->txrate.he_dcm = txrate->he_dcm; sinfo->txrate.he_ru_alloc = txrate->he_ru_alloc; + sinfo->txrate.eht_gi = txrate->eht_gi; } sinfo->txrate.flags = txrate->flags; sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BITRATE); @@ -1388,6 +1411,44 @@ mt7996_set_radar_background(struct ieee80211_hw *hw, return ret; } +#ifdef CONFIG_NET_MEDIATEK_SOC_WED +static int +mt7996_net_fill_forward_path(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_sta *sta, + struct net_device_path_ctx *ctx, + struct net_device_path *path) +{ + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv; + struct mt7996_dev *dev = mt7996_hw_dev(hw); + struct mt7996_phy *phy = mt7996_hw_phy(hw); + struct mtk_wed_device *wed = &dev->mt76.mmio.wed; + + if (phy != &dev->phy && phy->mt76->band_idx == MT_BAND2) + wed = &dev->mt76.mmio.wed_hif2; + + if (!mtk_wed_device_active(wed)) + return -ENODEV; + + if (msta->wcid.idx > MT7996_WTBL_STA) + return -EIO; + + path->type = DEV_PATH_MTK_WDMA; + path->dev = ctx->dev; + path->mtk_wdma.wdma_idx = wed->wdma_idx; + path->mtk_wdma.bss = mvif->mt76.idx; + path->mtk_wdma.queue = 0; + path->mtk_wdma.wcid = msta->wcid.idx; + + path->mtk_wdma.amsdu = mtk_wed_is_amsdu_supported(wed); + ctx->dev = NULL; + + return 0; +} + +#endif + const struct ieee80211_ops mt7996_ops = { .tx = mt7996_tx, .start = mt7996_start, @@ -1432,4 +1493,8 @@ const struct ieee80211_ops mt7996_ops = { .sta_add_debugfs = mt7996_sta_add_debugfs, #endif .set_radar_background = mt7996_set_radar_background, +#ifdef CONFIG_NET_MEDIATEK_SOC_WED + .net_fill_forward_path = mt7996_net_fill_forward_path, + .net_setup_tc = mt76_net_setup_tc, +#endif }; diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c index bf917beb943961..3c729b563edc5d 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c @@ -10,6 +10,20 @@ #include "mac.h" #include "eeprom.h" +#define fw_name(_dev, name, ...) ({ \ + char *_fw; \ + switch (mt76_chip(&(_dev)->mt76)) { \ + case 0x7992: \ + _fw = MT7992_##name; \ + break; \ + case 0x7990: \ + default: \ + _fw = MT7996_##name; \ + break; \ + } \ + _fw; \ +}) + struct mt7996_patch_hdr { char build_date[16]; char platform[4]; @@ -449,6 +463,43 @@ mt7996_mcu_ie_countdown(struct mt7996_dev *dev, struct sk_buff *skb) } } +static int +mt7996_mcu_update_tx_gi(struct rate_info *rate, struct all_sta_trx_rate *mcu_rate) +{ + switch (mcu_rate->tx_mode) { + case MT_PHY_TYPE_CCK: + case MT_PHY_TYPE_OFDM: + break; + case MT_PHY_TYPE_HT: + case MT_PHY_TYPE_HT_GF: + case MT_PHY_TYPE_VHT: + if (mcu_rate->tx_gi) + rate->flags |= RATE_INFO_FLAGS_SHORT_GI; + else + rate->flags &= ~RATE_INFO_FLAGS_SHORT_GI; + break; + case MT_PHY_TYPE_HE_SU: + case MT_PHY_TYPE_HE_EXT_SU: + case MT_PHY_TYPE_HE_TB: + case MT_PHY_TYPE_HE_MU: + if (mcu_rate->tx_gi > NL80211_RATE_INFO_HE_GI_3_2) + return -EINVAL; + rate->he_gi = mcu_rate->tx_gi; + break; + case MT_PHY_TYPE_EHT_SU: + case MT_PHY_TYPE_EHT_TRIG: + case MT_PHY_TYPE_EHT_MU: + if (mcu_rate->tx_gi > NL80211_RATE_INFO_EHT_GI_3_2) + return -EINVAL; + rate->eht_gi = mcu_rate->tx_gi; + break; + default: + return -EINVAL; + } + + return 0; +} + static void mt7996_mcu_rx_all_sta_info_event(struct mt7996_dev *dev, struct sk_buff *skb) { @@ -465,6 +516,16 @@ mt7996_mcu_rx_all_sta_info_event(struct mt7996_dev *dev, struct sk_buff *skb) struct mt76_wcid *wcid; switch (le16_to_cpu(res->tag)) { + case UNI_ALL_STA_TXRX_RATE: + wlan_idx = le16_to_cpu(res->rate[i].wlan_idx); + wcid = rcu_dereference(dev->mt76.wcid[wlan_idx]); + + if (!wcid) + break; + + if (mt7996_mcu_update_tx_gi(&wcid->rate, &res->rate[i])) + dev_err(dev->mt76.dev, "Failed to update TX GI\n"); + break; case UNI_ALL_STA_TXRX_ADM_STAT: wlan_idx = le16_to_cpu(res->adm_stat[i].wlan_idx); wcid = rcu_dereference(dev->mt76.wcid[wlan_idx]); @@ -497,6 +558,34 @@ mt7996_mcu_rx_all_sta_info_event(struct mt7996_dev *dev, struct sk_buff *skb) } } +static void +mt7996_mcu_rx_thermal_notify(struct mt7996_dev *dev, struct sk_buff *skb) +{ +#define THERMAL_NOTIFY_TAG 0x4 +#define THERMAL_NOTIFY 0x2 + struct mt76_phy *mphy = &dev->mt76.phy; + struct mt7996_mcu_thermal_notify *n; + struct mt7996_phy *phy; + + n = (struct mt7996_mcu_thermal_notify *)skb->data; + + if (le16_to_cpu(n->tag) != THERMAL_NOTIFY_TAG) + return; + + if (n->event_id != THERMAL_NOTIFY) + return; + + if (n->band_idx > MT_BAND2) + return; + + mphy = dev->mt76.phys[n->band_idx]; + if (!mphy) + return; + + phy = (struct mt7996_phy *)mphy->priv; + phy->throttle_state = n->duty_percent; +} + static void mt7996_mcu_rx_ext_event(struct mt7996_dev *dev, struct sk_buff *skb) { @@ -520,12 +609,82 @@ mt7996_mcu_rx_unsolicited_event(struct mt7996_dev *dev, struct sk_buff *skb) case MCU_EVENT_EXT: mt7996_mcu_rx_ext_event(dev, skb); break; + case MCU_UNI_EVENT_THERMAL: + mt7996_mcu_rx_thermal_notify(dev, skb); + break; default: break; } dev_kfree_skb(skb); } +static void +mt7996_mcu_wed_rro_event(struct mt7996_dev *dev, struct sk_buff *skb) +{ + struct mt7996_mcu_wed_rro_event *event = (void *)skb->data; + + if (!dev->has_rro) + return; + + skb_pull(skb, sizeof(struct mt7996_mcu_rxd) + 4); + + switch (le16_to_cpu(event->tag)) { + case UNI_WED_RRO_BA_SESSION_STATUS: { + struct mt7996_mcu_wed_rro_ba_event *e; + + while (skb->len >= sizeof(*e)) { + struct mt76_rx_tid *tid; + struct mt76_wcid *wcid; + u16 idx; + + e = (void *)skb->data; + idx = le16_to_cpu(e->wlan_id); + if (idx >= ARRAY_SIZE(dev->mt76.wcid)) + break; + + wcid = rcu_dereference(dev->mt76.wcid[idx]); + if (!wcid || !wcid->sta) + break; + + if (e->tid >= ARRAY_SIZE(wcid->aggr)) + break; + + tid = rcu_dereference(wcid->aggr[e->tid]); + if (!tid) + break; + + tid->id = le16_to_cpu(e->id); + skb_pull(skb, sizeof(*e)); + } + break; + } + case UNI_WED_RRO_BA_SESSION_DELETE: { + struct mt7996_mcu_wed_rro_ba_delete_event *e; + + while (skb->len >= sizeof(*e)) { + struct mt7996_wed_rro_session_id *session; + + e = (void *)skb->data; + session = kzalloc(sizeof(*session), GFP_ATOMIC); + if (!session) + break; + + session->id = le16_to_cpu(e->session_id); + + spin_lock_bh(&dev->wed_rro.lock); + list_add_tail(&session->list, &dev->wed_rro.poll_list); + spin_unlock_bh(&dev->wed_rro.lock); + + ieee80211_queue_work(mt76_hw(dev), &dev->wed_rro.work); + skb_pull(skb, sizeof(*e)); + } + break; + } + default: + break; + } +} + static void mt7996_mcu_uni_rx_unsolicited_event(struct mt7996_dev *dev, struct sk_buff *skb) { @@ -544,6 +703,9 @@ mt7996_mcu_uni_rx_unsolicited_event(struct mt7996_dev *dev, struct sk_buff *skb) case MCU_UNI_EVENT_ALL_STA_INFO: mt7996_mcu_rx_all_sta_info_event(dev, skb); break; + case MCU_UNI_EVENT_WED_RRO: + mt7996_mcu_wed_rro_event(dev, skb); + break; default: break; } @@ -963,7 +1125,7 @@ int mt7996_mcu_set_timing(struct mt7996_phy *phy, struct ieee80211_vif *vif) } static int -mt7996_mcu_sta_ba(struct mt76_dev *dev, struct mt76_vif *mvif, +mt7996_mcu_sta_ba(struct mt7996_dev *dev, struct mt76_vif *mvif, struct ieee80211_ampdu_params *params, bool enable, bool tx) { @@ -972,7 +1134,7 @@ mt7996_mcu_sta_ba(struct mt76_dev *dev, struct mt76_vif *mvif, struct sk_buff *skb; struct tlv *tlv; - skb = __mt76_connac_mcu_alloc_sta_req(dev, mvif, wcid, + skb = __mt76_connac_mcu_alloc_sta_req(&dev->mt76, mvif, wcid, MT7996_STA_UPDATE_MAX_SIZE); if (IS_ERR(skb)) return PTR_ERR(skb); @@ -986,8 +1148,9 @@ mt7996_mcu_sta_ba(struct mt76_dev *dev, struct mt76_vif *mvif, ba->ba_en = enable << params->tid; ba->amsdu = params->amsdu; ba->tid = params->tid; + ba->ba_rdd_rro = !tx && enable && dev->has_rro; - return mt76_mcu_skb_send_msg(dev, skb, + return mt76_mcu_skb_send_msg(&dev->mt76, skb, MCU_WMWA_UNI_CMD(STA_REC_UPDATE), true); } @@ -1002,8 +1165,7 @@ int mt7996_mcu_add_tx_ba(struct mt7996_dev *dev, if (enable && !params->amsdu) msta->wcid.amsdu = false; - return mt7996_mcu_sta_ba(&dev->mt76, &mvif->mt76, params, - enable, true); + return mt7996_mcu_sta_ba(dev, &mvif->mt76, params, enable, true); } int mt7996_mcu_add_rx_ba(struct mt7996_dev *dev, @@ -1013,8 +1175,7 @@ int mt7996_mcu_add_rx_ba(struct mt7996_dev *dev, struct mt7996_sta *msta = (struct mt7996_sta *)params->sta->drv_priv; struct mt7996_vif *mvif = msta->vif; - return mt7996_mcu_sta_ba(&dev->mt76, &mvif->mt76, params, - enable, false); + return mt7996_mcu_sta_ba(dev, &mvif->mt76, params, enable, false); } static void @@ -1108,7 +1269,7 @@ mt7996_mcu_sta_eht_tlv(struct sk_buff *skb, struct ieee80211_sta *sta) static void mt7996_mcu_sta_ht_tlv(struct sk_buff *skb, struct ieee80211_sta *sta) { - struct sta_rec_ht *ht; + struct sta_rec_ht_uni *ht; struct tlv *tlv; if (!sta->deflink.ht_cap.ht_supported) @@ -1116,8 +1277,12 @@ mt7996_mcu_sta_ht_tlv(struct sk_buff *skb, struct ieee80211_sta *sta) tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_HT, sizeof(*ht)); - ht = (struct sta_rec_ht *)tlv; + ht = (struct sta_rec_ht_uni *)tlv; ht->ht_cap = cpu_to_le16(sta->deflink.ht_cap.cap); + ht->ampdu_param = u8_encode_bits(sta->deflink.ht_cap.ampdu_factor, + IEEE80211_HT_AMPDU_PARM_FACTOR) | + u8_encode_bits(sta->deflink.ht_cap.ampdu_density, + IEEE80211_HT_AMPDU_PARM_DENSITY); } static void @@ -1574,44 +1739,6 @@ mt7996_mcu_sta_bfee_tlv(struct mt7996_dev *dev, struct sk_buff *skb, bfee->fb_identity_matrix = (nrow == 1 && tx_ant == 2); } -static void -mt7996_mcu_sta_phy_tlv(struct mt7996_dev *dev, struct sk_buff *skb, - struct ieee80211_vif *vif, struct ieee80211_sta *sta) -{ - struct sta_rec_phy *phy; - struct tlv *tlv; - u8 af = 0, mm = 0; - - if (!sta->deflink.ht_cap.ht_supported && !sta->deflink.he_6ghz_capa.capa) - return; - - tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_PHY, sizeof(*phy)); - - phy = (struct sta_rec_phy *)tlv; - if (sta->deflink.ht_cap.ht_supported) { - af = sta->deflink.ht_cap.ampdu_factor; - mm = sta->deflink.ht_cap.ampdu_density; - } - - if (sta->deflink.vht_cap.vht_supported) { - u8 vht_af = FIELD_GET(IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_MASK, - sta->deflink.vht_cap.cap); - - af = max_t(u8, af, vht_af); - } - - if (sta->deflink.he_6ghz_capa.capa) { - af = le16_get_bits(sta->deflink.he_6ghz_capa.capa, - IEEE80211_HE_6GHZ_CAP_MAX_AMPDU_LEN_EXP); - mm = le16_get_bits(sta->deflink.he_6ghz_capa.capa, - IEEE80211_HE_6GHZ_CAP_MIN_MPDU_START); - } - - phy->ampdu = FIELD_PREP(IEEE80211_HT_AMPDU_PARM_FACTOR, af) | - FIELD_PREP(IEEE80211_HT_AMPDU_PARM_DENSITY, mm); - phy->max_ampdu_len = af; -} - static void mt7996_mcu_sta_hdrt_tlv(struct mt7996_dev *dev, struct sk_buff *skb) { @@ -1700,14 +1827,13 @@ int mt7996_mcu_set_fixed_rate_ctrl(struct mt7996_dev *dev, MCU_WM_UNI_CMD(RA), true); } -static int -mt7996_mcu_set_fixed_field(struct mt7996_dev *dev, struct ieee80211_vif *vif, - struct ieee80211_sta *sta, void *data, u32 field) +int mt7996_mcu_set_fixed_field(struct mt7996_dev *dev, struct ieee80211_vif *vif, + struct ieee80211_sta *sta, void *data, u32 field) { struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv; - struct sta_phy *phy = data; - struct sta_rec_ra_fixed *ra; + struct sta_phy_uni *phy = data; + struct sta_rec_ra_fixed_uni *ra; struct sk_buff *skb; struct tlv *tlv; @@ -1718,7 +1844,7 @@ mt7996_mcu_set_fixed_field(struct mt7996_dev *dev, struct ieee80211_vif *vif, return PTR_ERR(skb); tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_RA_UPDATE, sizeof(*ra)); - ra = (struct sta_rec_ra_fixed *)tlv; + ra = (struct sta_rec_ra_fixed_uni *)tlv; switch (field) { case RATE_PARAM_AUTO: @@ -1730,6 +1856,9 @@ mt7996_mcu_set_fixed_field(struct mt7996_dev *dev, struct ieee80211_vif *vif, if (phy) ra->phy = *phy; break; + case RATE_PARAM_MMPS_UPDATE: + ra->mmps_mode = mt7996_mcu_get_mmps_mode(sta->deflink.smps_mode); + break; default: break; } @@ -1747,7 +1876,7 @@ mt7996_mcu_add_rate_ctrl_fixed(struct mt7996_dev *dev, struct ieee80211_vif *vif struct cfg80211_chan_def *chandef = &mvif->phy->mt76->chandef; struct cfg80211_bitrate_mask *mask = &mvif->bitrate_mask; enum nl80211_band band = chandef->chan->band; - struct sta_phy phy = {}; + struct sta_phy_uni phy = {}; int ret, nrates = 0; #define __sta_phy_bitrate_mask_check(_mcs, _gi, _ht, _he) \ @@ -1835,13 +1964,13 @@ mt7996_mcu_sta_rate_ctrl_tlv(struct sk_buff *skb, struct mt7996_dev *dev, struct cfg80211_chan_def *chandef = &mphy->chandef; struct cfg80211_bitrate_mask *mask = &mvif->bitrate_mask; enum nl80211_band band = chandef->chan->band; - struct sta_rec_ra *ra; + struct sta_rec_ra_uni *ra; struct tlv *tlv; u32 supp_rate = sta->deflink.supp_rates[band]; u32 cap = sta->wme ? STA_CAP_WMM : 0; tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_RA, sizeof(*ra)); - ra = (struct sta_rec_ra *)tlv; + ra = (struct sta_rec_ra_uni *)tlv; ra->valid = true; ra->auto_rate = true; @@ -2018,8 +2147,6 @@ int mt7996_mcu_add_sta(struct mt7996_dev *dev, struct ieee80211_vif *vif, /* tag order is in accordance with firmware dependency. */ if (sta) { - /* starec phy */ - mt7996_mcu_sta_phy_tlv(dev, skb, vif, sta); /* starec hdrt mode */ mt7996_mcu_sta_hdrt_tlv(dev, skb); /* starec bfer */ @@ -2058,7 +2185,6 @@ int mt7996_mcu_add_sta(struct mt7996_dev *dev, struct ieee80211_vif *vif, static int mt7996_mcu_sta_key_tlv(struct mt76_wcid *wcid, - struct mt76_connac_sta_key_conf *sta_key_conf, struct sk_buff *skb, struct ieee80211_key_conf *key, enum set_key_cmd cmd) @@ -2079,43 +2205,22 @@ mt7996_mcu_sta_key_tlv(struct mt76_wcid *wcid, return -EOPNOTSUPP; sec_key = &sec->key[0]; + sec_key->wlan_idx = cpu_to_le16(wcid->idx); + sec_key->mgmt_prot = 0; + sec_key->cipher_id = cipher; sec_key->cipher_len = sizeof(*sec_key); - - if (cipher == MCU_CIPHER_BIP_CMAC_128) { - sec_key->wlan_idx = cpu_to_le16(wcid->idx); - sec_key->cipher_id = MCU_CIPHER_AES_CCMP; - sec_key->key_id = sta_key_conf->keyidx; - sec_key->key_len = 16; - memcpy(sec_key->key, sta_key_conf->key, 16); - - sec_key = &sec->key[1]; - sec_key->wlan_idx = cpu_to_le16(wcid->idx); - sec_key->cipher_id = MCU_CIPHER_BIP_CMAC_128; - sec_key->cipher_len = sizeof(*sec_key); - sec_key->key_len = 16; - memcpy(sec_key->key, key->key, 16); - sec->n_cipher = 2; - } else { - sec_key->wlan_idx = cpu_to_le16(wcid->idx); - sec_key->cipher_id = cipher; - sec_key->key_id = key->keyidx; - sec_key->key_len = key->keylen; - memcpy(sec_key->key, key->key, key->keylen); - - if (cipher == MCU_CIPHER_TKIP) { - /* Rx/Tx MIC keys are swapped */ - memcpy(sec_key->key + 16, key->key + 24, 8); - memcpy(sec_key->key + 24, key->key + 16, 8); - } - - /* store key_conf for BIP batch update */ - if (cipher == MCU_CIPHER_AES_CCMP) { - memcpy(sta_key_conf->key, key->key, key->keylen); - sta_key_conf->keyidx = key->keyidx; - } - - sec->n_cipher = 1; + sec_key->key_id = key->keyidx; + sec_key->key_len = key->keylen; + sec_key->need_resp = 0; + memcpy(sec_key->key, key->key, key->keylen); + + if (cipher == MCU_CIPHER_TKIP) { + /* Rx/Tx MIC keys are swapped */ + memcpy(sec_key->key + 16, key->key + 24, 8); + memcpy(sec_key->key + 24, key->key + 16, 8); } + + sec->n_cipher = 1; } else { sec->n_cipher = 0; } @@ -2124,7 +2229,6 @@ mt7996_mcu_sta_key_tlv(struct mt76_wcid *wcid, } int mt7996_mcu_add_key(struct mt76_dev *dev, struct ieee80211_vif *vif, - struct mt76_connac_sta_key_conf *sta_key_conf, struct ieee80211_key_conf *key, int mcu_cmd, struct mt76_wcid *wcid, enum set_key_cmd cmd) { @@ -2137,13 +2241,99 @@ int mt7996_mcu_add_key(struct mt76_dev *dev, struct ieee80211_vif *vif, if (IS_ERR(skb)) return PTR_ERR(skb); - ret = mt7996_mcu_sta_key_tlv(wcid, sta_key_conf, skb, key, cmd); + ret = mt7996_mcu_sta_key_tlv(wcid, skb, key, cmd); if (ret) return ret; return mt76_mcu_skb_send_msg(dev, skb, mcu_cmd, true); } +static int mt7996_mcu_get_pn(struct mt7996_dev *dev, struct ieee80211_vif *vif, + u8 *pn) +{ +#define TSC_TYPE_BIGTK_PN 2 + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + struct sta_rec_pn_info *pn_info; + struct sk_buff *skb, *rskb; + struct tlv *tlv; + int ret; + + skb = mt76_connac_mcu_alloc_sta_req(&dev->mt76, &mvif->mt76, &mvif->sta.wcid); + if (IS_ERR(skb)) + return PTR_ERR(skb); + + tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_PN_INFO, sizeof(*pn_info)); + pn_info = (struct sta_rec_pn_info *)tlv; + + pn_info->tsc_type = TSC_TYPE_BIGTK_PN; + ret = mt76_mcu_skb_send_and_get_msg(&dev->mt76, skb, + MCU_WM_UNI_CMD_QUERY(STA_REC_UPDATE), + true, &rskb); + if (ret) + return ret; + + skb_pull(rskb, 4); + + pn_info = (struct sta_rec_pn_info *)rskb->data; + if (le16_to_cpu(pn_info->tag) == STA_REC_PN_INFO) + memcpy(pn, pn_info->pn, 6); + + dev_kfree_skb(rskb); + return 0; +} + +int mt7996_mcu_bcn_prot_enable(struct mt7996_dev *dev, struct ieee80211_vif *vif, + struct ieee80211_key_conf *key) +{ + struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv; + struct mt7996_mcu_bcn_prot_tlv *bcn_prot; + struct sk_buff *skb; + struct tlv *tlv; + u8 pn[6] = {}; + int len = sizeof(struct bss_req_hdr) + + sizeof(struct mt7996_mcu_bcn_prot_tlv); + int ret; + + skb = __mt7996_mcu_alloc_bss_req(&dev->mt76, &mvif->mt76, len); + if (IS_ERR(skb)) + return PTR_ERR(skb); + + tlv = mt76_connac_mcu_add_tlv(skb, UNI_BSS_INFO_BCN_PROT, sizeof(*bcn_prot)); + + bcn_prot = (struct mt7996_mcu_bcn_prot_tlv *)tlv; + + ret = mt7996_mcu_get_pn(dev, vif, pn); + if (ret) { + dev_kfree_skb(skb); + return ret; + } + + switch (key->cipher) { + case WLAN_CIPHER_SUITE_AES_CMAC: + bcn_prot->cipher_id = MCU_CIPHER_BCN_PROT_CMAC_128; + break; + case WLAN_CIPHER_SUITE_BIP_GMAC_128: + bcn_prot->cipher_id = MCU_CIPHER_BCN_PROT_GMAC_128; + break; + case WLAN_CIPHER_SUITE_BIP_GMAC_256: + bcn_prot->cipher_id = MCU_CIPHER_BCN_PROT_GMAC_256; + break; + case WLAN_CIPHER_SUITE_BIP_CMAC_256: + default: + dev_err(dev->mt76.dev, "Not supported Bigtk Cipher\n"); + dev_kfree_skb(skb); + return -EOPNOTSUPP; + } + + pn[0]++; + memcpy(bcn_prot->pn, pn, 6); + bcn_prot->enable = BP_SW_MODE; + memcpy(bcn_prot->key, key->key, WLAN_MAX_KEY_LEN); + bcn_prot->key_id = key->keyidx; + + return mt76_mcu_skb_send_msg(&dev->mt76, skb, + MCU_WMWA_UNI_CMD(BSS_INFO_UPDATE), true); +} int mt7996_mcu_add_dev_info(struct mt7996_phy *phy, struct ieee80211_vif *vif, bool enable) { @@ -2463,7 +2653,7 @@ static int mt7996_load_patch(struct mt7996_dev *dev) return -EAGAIN; } - ret = request_firmware(&fw, MT7996_ROM_PATCH, dev->mt76.dev); + ret = request_firmware(&fw, fw_name(dev, ROM_PATCH), dev->mt76.dev); if (ret) goto out; @@ -2626,17 +2816,17 @@ static int mt7996_load_ram(struct mt7996_dev *dev) { int ret; - ret = __mt7996_load_ram(dev, "WM", MT7996_FIRMWARE_WM, + ret = __mt7996_load_ram(dev, "WM", fw_name(dev, FIRMWARE_WM), MT7996_RAM_TYPE_WM); if (ret) return ret; - ret = __mt7996_load_ram(dev, "DSP", MT7996_FIRMWARE_DSP, + ret = __mt7996_load_ram(dev, "DSP", fw_name(dev, FIRMWARE_DSP), MT7996_RAM_TYPE_DSP); if (ret) return ret; - return __mt7996_load_ram(dev, "WA", MT7996_FIRMWARE_WA, + return __mt7996_load_ram(dev, "WA", fw_name(dev, FIRMWARE_WA), MT7996_RAM_TYPE_WA); } @@ -2788,9 +2978,10 @@ mt7996_mcu_init_rx_airtime(struct mt7996_dev *dev) { struct uni_header hdr = {}; struct sk_buff *skb; - int len, num; + int len, num, i; - num = 2 + 2 * (dev->dbdc_support + dev->tbtc_support); + num = 2 + 2 * (mt7996_band_valid(dev, MT_BAND1) + + mt7996_band_valid(dev, MT_BAND2)); len = sizeof(hdr) + num * sizeof(struct vow_rx_airtime); skb = mt76_mcu_msg_alloc(&dev->mt76, NULL, len); if (!skb) @@ -2798,13 +2989,10 @@ mt7996_mcu_init_rx_airtime(struct mt7996_dev *dev) skb_put_data(skb, &hdr, sizeof(hdr)); - mt7996_add_rx_airtime_tlv(skb, dev->mt76.phy.band_idx); - - if (dev->dbdc_support) - mt7996_add_rx_airtime_tlv(skb, MT_BAND1); - - if (dev->tbtc_support) - mt7996_add_rx_airtime_tlv(skb, MT_BAND2); + for (i = 0; i < __MT_MAX_BAND; i++) { + if (mt7996_band_valid(dev, i)) + mt7996_add_rx_airtime_tlv(skb, i); + } return mt76_mcu_skb_send_msg(&dev->mt76, skb, MCU_WM_UNI_CMD(VOW), true); @@ -3230,7 +3418,7 @@ int mt7996_mcu_set_chan_info(struct mt7996_phy *phy, u16 tag) .center_ch = ieee80211_frequency_to_channel(freq1), .bw = mt76_connac_chan_bw(chandef), .tx_path_num = hweight16(phy->mt76->chainmask), - .rx_path = phy->mt76->chainmask >> dev->chainshift[band_idx], + .rx_path = mt7996_rx_chainmask(phy) >> dev->chainshift[band_idx], .band_idx = band_idx, .channel_band = ch_band[chandef->chan->band], }; @@ -3502,6 +3690,121 @@ int mt7996_mcu_get_chan_mib_info(struct mt7996_phy *phy, bool chan_switch) return 0; } +int mt7996_mcu_get_temperature(struct mt7996_phy *phy) +{ +#define TEMPERATURE_QUERY 0 +#define GET_TEMPERATURE 0 + struct { + u8 _rsv[4]; + + __le16 tag; + __le16 len; + + u8 rsv1; + u8 action; + u8 band_idx; + u8 rsv2; + } req = { + .tag = cpu_to_le16(TEMPERATURE_QUERY), + .len = cpu_to_le16(sizeof(req) - 4), + .action = GET_TEMPERATURE, + .band_idx = phy->mt76->band_idx, + }; + struct mt7996_mcu_thermal { + u8 _rsv[4]; + + __le16 tag; + __le16 len; + + __le32 rsv; + __le32 temperature; + } __packed * res; + struct sk_buff *skb; + int ret; + + ret = mt76_mcu_send_and_get_msg(&phy->dev->mt76, MCU_WM_UNI_CMD(THERMAL), + &req, sizeof(req), true, &skb); + if (ret) + return ret; + + res = (void *)skb->data; + + return le32_to_cpu(res->temperature); +} + +int mt7996_mcu_set_thermal_throttling(struct mt7996_phy *phy, u8 state) +{ + struct { + u8 _rsv[4]; + + __le16 tag; + __le16 len; + + struct mt7996_mcu_thermal_ctrl ctrl; + } __packed req = { + .tag = cpu_to_le16(UNI_CMD_THERMAL_PROTECT_DUTY_CONFIG), + .len = cpu_to_le16(sizeof(req) - 4), + .ctrl = { + .band_idx = phy->mt76->band_idx, + }, + }; + int level, ret; + + /* set duty cycle and level */ + for (level = 0; level < 4; level++) { + req.ctrl.duty.duty_level = level; + req.ctrl.duty.duty_cycle = state; + state /= 2; + + ret = mt76_mcu_send_msg(&phy->dev->mt76, MCU_WM_UNI_CMD(THERMAL), + &req, sizeof(req), false); + if (ret) + return ret; + } + + return 0; +} + +int mt7996_mcu_set_thermal_protect(struct mt7996_phy *phy, bool enable) +{ +#define SUSTAIN_PERIOD 10 + struct { + u8 _rsv[4]; + + __le16 tag; + __le16 len; + + struct mt7996_mcu_thermal_ctrl ctrl; + struct mt7996_mcu_thermal_enable enable; + } __packed req = { + .len = cpu_to_le16(sizeof(req) - 4 - sizeof(req.enable)), + .ctrl = { + .band_idx = phy->mt76->band_idx, + .type.protect_type = 1, + .type.trigger_type = 1, + }, + }; + int ret; + + req.tag = cpu_to_le16(UNI_CMD_THERMAL_PROTECT_DISABLE); + + ret = mt76_mcu_send_msg(&phy->dev->mt76, MCU_WM_UNI_CMD(THERMAL), + &req, sizeof(req) - sizeof(req.enable), false); + if (ret || !enable) + return ret; + + /* set high-temperature trigger threshold */ + req.tag = cpu_to_le16(UNI_CMD_THERMAL_PROTECT_ENABLE); + req.enable.restore_temp = cpu_to_le32(phy->throttle_temp[0]); + req.enable.trigger_temp = cpu_to_le32(phy->throttle_temp[1]); + req.enable.sustain_time = cpu_to_le16(SUSTAIN_PERIOD); + + req.len = cpu_to_le16(sizeof(req) - 4); + + return mt76_mcu_send_msg(&phy->dev->mt76, MCU_WM_UNI_CMD(THERMAL), + &req, sizeof(req), false); +} + int mt7996_mcu_set_ser(struct mt7996_dev *dev, u8 action, u8 val, u8 band) { struct { @@ -3964,6 +4267,35 @@ int mt7996_mcu_wtbl_update_hdr_trans(struct mt7996_dev *dev, MCU_WMWA_UNI_CMD(STA_REC_UPDATE), true); } +int mt7996_mcu_set_fixed_rate_table(struct mt7996_phy *phy, u8 table_idx, + u16 rate_idx, bool beacon) +{ +#define UNI_FIXED_RATE_TABLE_SET 0 +#define SPE_IXD_SELECT_TXD 0 +#define SPE_IXD_SELECT_BMC_WTBL 1 + struct mt7996_dev *dev = phy->dev; + struct fixed_rate_table_ctrl req = { + .tag = cpu_to_le16(UNI_FIXED_RATE_TABLE_SET), + .len = cpu_to_le16(sizeof(req) - 4), + .table_idx = table_idx, + .rate_idx = cpu_to_le16(rate_idx), + .gi = 1, + .he_ltf = 1, + }; + u8 band_idx = phy->mt76->band_idx; + + if (beacon) { + req.spe_idx_sel = SPE_IXD_SELECT_TXD; + req.spe_idx = 24 + band_idx; + phy->beacon_rate = rate_idx; + } else { + req.spe_idx_sel = SPE_IXD_SELECT_BMC_WTBL; + } + + return mt76_mcu_send_msg(&dev->mt76, MCU_WM_UNI_CMD(FIXED_RATE_TABLE), + &req, sizeof(req), false); +} + int mt7996_mcu_rf_regval(struct mt7996_dev *dev, u32 regidx, u32 *val, bool set) { struct { @@ -4019,14 +4351,12 @@ int mt7996_mcu_trigger_assert(struct mt7996_dev *dev) &req, sizeof(req), false); } -int mt7996_mcu_set_rro(struct mt7996_dev *dev, u16 tag, u8 val) +int mt7996_mcu_set_rro(struct mt7996_dev *dev, u16 tag, u16 val) { struct { u8 __rsv1[4]; - __le16 tag; __le16 len; - union { struct { u8 type; @@ -4041,6 +4371,11 @@ int mt7996_mcu_set_rro(struct mt7996_dev *dev, u16 tag, u8 val) u8 path; u8 __rsv2[3]; } __packed txfree_path; + struct { + __le16 flush_one; + __le16 flush_all; + u8 __rsv2[4]; + } __packed timeout; }; } __packed req = { .tag = cpu_to_le16(tag), @@ -4057,6 +4392,10 @@ int mt7996_mcu_set_rro(struct mt7996_dev *dev, u16 tag, u8 val) case UNI_RRO_SET_TXFREE_PATH: req.txfree_path.path = val; break; + case UNI_RRO_SET_FLUSH_TIMEOUT: + req.timeout.flush_one = cpu_to_le16(val); + req.timeout.flush_all = cpu_to_le16(2 * val); + break; default: return -EINVAL; } @@ -4081,3 +4420,80 @@ int mt7996_mcu_get_all_sta_info(struct mt7996_phy *phy, u16 tag) return mt76_mcu_send_msg(&dev->mt76, MCU_WM_UNI_CMD(ALL_STA_INFO), &req, sizeof(req), false); } + +int mt7996_mcu_wed_rro_reset_sessions(struct mt7996_dev *dev, u16 id) +{ + struct { + u8 __rsv[4]; + + __le16 tag; + __le16 len; + __le16 session_id; + u8 pad[4]; + } __packed req = { + .tag = cpu_to_le16(UNI_RRO_DEL_BA_SESSION), + .len = cpu_to_le16(sizeof(req) - 4), + .session_id = cpu_to_le16(id), + }; + + return mt76_mcu_send_msg(&dev->mt76, MCU_WM_UNI_CMD(RRO), &req, + sizeof(req), true); +} + +int mt7996_mcu_set_txpower_sku(struct mt7996_phy *phy) +{ +#define TX_POWER_LIMIT_TABLE_RATE 0 + struct mt7996_dev *dev = phy->dev; + struct mt76_phy *mphy = phy->mt76; + struct ieee80211_hw *hw = mphy->hw; + struct tx_power_limit_table_ctrl { + u8 __rsv1[4]; + + __le16 tag; + __le16 len; + u8 power_ctrl_id; + u8 power_limit_type; + u8 band_idx; + } __packed req = { + .tag = cpu_to_le16(UNI_TXPOWER_POWER_LIMIT_TABLE_CTRL), + .len = cpu_to_le16(sizeof(req) + MT7996_SKU_RATE_NUM - 4), + .power_ctrl_id = UNI_TXPOWER_POWER_LIMIT_TABLE_CTRL, + .power_limit_type = TX_POWER_LIMIT_TABLE_RATE, + .band_idx = phy->mt76->band_idx, + }; + struct mt76_power_limits la = {}; + struct sk_buff *skb; + int i, tx_power; + + tx_power = mt7996_get_power_bound(phy, hw->conf.power_level); + tx_power = mt76_get_rate_power_limits(mphy, mphy->chandef.chan, + &la, tx_power); + mphy->txpower_cur = tx_power; + + skb = mt76_mcu_msg_alloc(&dev->mt76, NULL, + sizeof(req) + MT7996_SKU_RATE_NUM); + if (!skb) + return -ENOMEM; + + skb_put_data(skb, &req, sizeof(req)); + /* cck and ofdm */ + skb_put_data(skb, &la.cck, sizeof(la.cck) + sizeof(la.ofdm)); + /* ht20 */ + skb_put_data(skb, &la.mcs[0], 8); + /* ht40 */ + skb_put_data(skb, &la.mcs[1], 9); + + /* vht */ + for (i = 0; i < 4; i++) { + skb_put_data(skb, &la.mcs[i], sizeof(la.mcs[i])); + skb_put_zero(skb, 2); /* padding */ + } + + /* he */ + skb_put_data(skb, &la.ru[0], sizeof(la.ru)); + /* eht */ + skb_put_data(skb, &la.eht[0], sizeof(la.eht)); + + return mt76_mcu_skb_send_msg(&dev->mt76, skb, + MCU_WM_UNI_CMD(TXPOWER), true); +} diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.h b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.h index a88f6af323dae0..36cacc495c75d3 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.h +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.h @@ -30,6 +30,28 @@ struct mt7996_mcu_uni_event { __le32 status; /* 0: success, others: fail */ } __packed; +struct mt7996_mcu_thermal_ctrl { + u8 ctrl_id; + u8 band_idx; + union { + struct { + u8 protect_type; /* 1: duty admit, 2: radio off */ + u8 trigger_type; /* 0: low, 1: high */ + } __packed type; + struct { + u8 duty_level; /* level 0~3 */ + u8 duty_cycle; + } __packed duty; + }; +} __packed; + +struct mt7996_mcu_thermal_enable { + __le32 trigger_temp; + __le32 restore_temp; + __le16 sustain_time; + u8 rsv[2]; +} __packed; + struct mt7996_mcu_csa_notify { struct mt7996_mcu_rxd rxd; @@ -153,6 +175,27 @@ struct mt7996_mcu_mib { __le64 data; } __packed; +struct all_sta_trx_rate { + __le16 wlan_idx; + u8 __rsv1[2]; + u8 tx_mode; + u8 flags; + u8 tx_stbc; + u8 tx_gi; + u8 tx_bw; + u8 tx_ldpc; + u8 tx_mcs; + u8 tx_nss; + u8 rx_rate; + u8 rx_mode; + u8 rx_nsts; + u8 rx_gi; + u8 rx_coding; + u8 rx_stbc; + u8 rx_bw; + u8 __rsv2; +} __packed; + struct mt7996_mcu_all_sta_info_event { u8 rsv[4]; __le16 tag; @@ -160,23 +203,75 @@ struct mt7996_mcu_all_sta_info_event { u8 more; u8 rsv2; __le16 sta_num; - u8 rsv3[2]; + u8 rsv3[4]; union { - struct { + DECLARE_FLEX_ARRAY(struct all_sta_trx_rate, rate); + DECLARE_FLEX_ARRAY(struct { __le16 wlan_idx; u8 rsv[2]; __le32 tx_bytes[IEEE80211_NUM_ACS]; __le32 rx_bytes[IEEE80211_NUM_ACS]; - } adm_stat[0]; + } __packed, adm_stat); - struct { + DECLARE_FLEX_ARRAY(struct { __le16 wlan_idx; u8 rsv[2]; __le32 tx_msdu_cnt; __le32 rx_msdu_cnt; - } msdu_cnt[0]; - }; + } __packed, msdu_cnt); + } __packed; +} __packed; + +struct mt7996_mcu_wed_rro_event { + struct mt7996_mcu_rxd rxd; + + u8 __rsv1[4]; + + __le16 tag; + __le16 len; +} __packed; + +struct mt7996_mcu_wed_rro_ba_event { + __le16 tag; + __le16 len; + + __le16 wlan_id; + u8 tid; + u8 __rsv1; + __le32 status; + __le16 id; + u8 __rsv2[2]; +} __packed; + +struct mt7996_mcu_wed_rro_ba_delete_event { + __le16 tag; + __le16 len; + + __le16 session_id; + u8 __rsv2[2]; +} __packed; + +enum { + UNI_WED_RRO_BA_SESSION_STATUS, + UNI_WED_RRO_BA_SESSION_TBL, + UNI_WED_RRO_BA_SESSION_DELETE, +}; + +struct mt7996_mcu_thermal_notify { + struct mt7996_mcu_rxd rxd; + + u8 __rsv1[4]; + + __le16 tag; + __le16 len; + + u8 event_id; + u8 band_idx; + u8 level_idx; + u8 duty_percent; + __le32 restore_temp; + u8 __rsv2[4]; } __packed; enum mt7996_chan_mib_offs { @@ -247,7 +342,24 @@ struct bss_rate_tlv { u8 short_preamble; u8 bc_fixed_rate; u8 mc_fixed_rate; - u8 __rsv2[1]; + u8 __rsv2[9]; +} __packed; + +enum { + BP_DISABLE, + BP_SW_MODE, + BP_HW_MODE, +}; + +struct mt7996_mcu_bcn_prot_tlv { + __le16 tag; + __le16 len; + u8 pn[6]; + u8 enable; + u8 cipher_id; + u8 key[WLAN_MAX_KEY_LEN]; + u8 key_id; + u8 __rsv[3]; } __packed; struct bss_ra_tlv { @@ -372,6 +484,15 @@ struct bss_mld_tlv { u8 __rsv[3]; } __packed; +struct sta_rec_ht_uni { + __le16 tag; + __le16 len; + __le16 ht_cap; + __le16 ht_cap_ext; + u8 ampdu_param; + u8 _rsv[3]; +} __packed; + struct sta_rec_ba_uni { __le16 tag; __le16 len; @@ -421,6 +542,73 @@ struct sta_rec_sec_uni { struct sec_key_uni key[2]; } __packed; +struct sta_phy_uni { + u8 type; + u8 flag; + u8 stbc; + u8 sgi; + u8 bw; + u8 ldpc; + u8 mcs; + u8 nss; + u8 he_ltf; + u8 rsv[3]; +}; + +struct sta_rec_ra_uni { + __le16 tag; + __le16 len; + + u8 valid; + u8 auto_rate; + u8 phy_mode; + u8 channel; + u8 bw; + u8 disable_cck; + u8 ht_mcs32; + u8 ht_gf; + u8 ht_mcs[4]; + u8 mmps_mode; + u8 gband_256; + u8 af; + u8 auth_wapi_mode; + u8 rate_len; + + u8 supp_mode; + u8 supp_cck_rate; + u8 supp_ofdm_rate; + __le32 supp_ht_mcs; + __le16 supp_vht_mcs[4]; + + u8 op_mode; + u8 op_vht_chan_width; + u8 op_vht_rx_nss; + u8 op_vht_rx_nss_type; + + __le32 sta_cap; + + struct sta_phy_uni phy; + u8 rx_rcpi[4]; +} __packed; + +struct sta_rec_ra_fixed_uni { + __le16 tag; + __le16 len; + + __le32 field; + u8 op_mode; + u8 op_vht_chan_width; + u8 op_vht_rx_nss; + u8 op_vht_rx_nss_type; + + struct sta_phy_uni phy; + + u8 spe_idx; + u8 short_preamble; + u8 is_5g; + u8 mmps_mode; +} __packed; + struct sta_rec_hdrt { __le16 tag; __le16 len; @@ -596,17 +784,16 @@ enum { #define MT7996_STA_UPDATE_MAX_SIZE (sizeof(struct sta_req_hdr) + \ sizeof(struct sta_rec_basic) + \ sizeof(struct sta_rec_bf) + \ - sizeof(struct sta_rec_ht) + \ + sizeof(struct sta_rec_ht_uni) + \ sizeof(struct sta_rec_he_v2) + \ sizeof(struct sta_rec_ba_uni) + \ sizeof(struct sta_rec_vht) + \ sizeof(struct sta_rec_uapsd) + \ sizeof(struct sta_rec_amsdu) + \ sizeof(struct sta_rec_bfee) + \ - sizeof(struct sta_rec_phy) + \ - sizeof(struct sta_rec_ra) + \ + sizeof(struct sta_rec_ra_uni) + \ sizeof(struct sta_rec_sec) + \ - sizeof(struct sta_rec_ra_fixed) + \ + sizeof(struct sta_rec_ra_fixed_uni) + \ sizeof(struct sta_rec_he_6g_capa) + \ sizeof(struct sta_rec_eht) + \ sizeof(struct sta_rec_hdrt) + \ @@ -622,6 +809,18 @@ enum { #define MT7996_MAX_BSS_OFFLOAD_SIZE (MT7996_MAX_BEACON_SIZE + \ MT7996_BEACON_UPDATE_SIZE) +static inline s8 +mt7996_get_power_bound(struct mt7996_phy *phy, s8 txpower) +{ + struct mt76_phy *mphy = phy->mt76; + int n_chains = hweight16(mphy->chainmask); + + txpower = mt76_get_sar_power(mphy, mphy->chandef.chan, txpower * 2); + txpower -= mt76_tx_power_nss_delta(n_chains); + + return txpower; +} + enum { UNI_BAND_CONFIG_RADIO_ENABLE, UNI_BAND_CONFIG_RTS_THRESHOLD = 0x08, @@ -669,6 +868,8 @@ enum { UNI_RRO_GET_BA_SESSION_TABLE, UNI_RRO_SET_BYPASS_MODE, UNI_RRO_SET_TXFREE_PATH, + UNI_RRO_DEL_BA_SESSION, + UNI_RRO_SET_FLUSH_TIMEOUT }; enum{ @@ -682,6 +883,16 @@ enum{ UNI_CMD_SR_SET_SIGA = 0xd0, }; +enum { + UNI_CMD_THERMAL_PROTECT_ENABLE = 0x6, + UNI_CMD_THERMAL_PROTECT_DISABLE, + UNI_CMD_THERMAL_PROTECT_DUTY_CONFIG, +}; + +enum { + UNI_TXPOWER_POWER_LIMIT_TABLE_CTRL = 4, +}; + enum { UNI_CMD_ACCESS_REG_BASIC = 0x0, UNI_CMD_ACCESS_RF_REG_BASIC, @@ -720,4 +931,24 @@ enum { #define MT7996_SEC_KEY_IDX GENMASK(2, 1) #define MT7996_SEC_IV BIT(3) +struct fixed_rate_table_ctrl { + u8 _rsv[4]; + + __le16 tag; + __le16 len; + + u8 table_idx; + u8 antenna_idx; + __le16 rate_idx; + u8 spe_idx_sel; + u8 spe_idx; + u8 gi; + u8 he_ltf; + bool ldpc; + bool txbf; + bool dynamic_bw; + + u8 _rsv2; +} __packed; + #endif diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mmio.c b/drivers/net/wireless/mediatek/mt76/mt7996/mmio.c index 3a591a7b47ae68..c50d89a445e956 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mmio.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mmio.c @@ -6,10 +6,16 @@ #include #include #include +#include #include "mt7996.h" #include "mac.h" +#include "mcu.h" #include "../trace.h" +#include "../dma.h" + +static bool wed_enable; +module_param(wed_enable, bool, 0644); static const struct __base mt7996_reg_base[] = { [WF_AGG_BASE] = { { 0x820e2000, 0x820f2000, 0x830e2000 } }, @@ -24,6 +30,58 @@ static const struct __base mt7996_reg_base[] = { [WF_RATE_BASE] = { { 0x820ee000, 0x820fe000, 0x830ee000 } }, }; +static const u32 mt7996_offs[] = { + [MIB_RVSR0] = 0x720, + [MIB_RVSR1] = 0x724, + [MIB_BTSCR5] = 0x788, + [MIB_BTSCR6] = 0x798, + [MIB_RSCR1] = 0x7ac, + [MIB_RSCR27] = 0x954, + [MIB_RSCR28] = 0x958, + [MIB_RSCR29] = 0x95c, + [MIB_RSCR30] = 0x960, + [MIB_RSCR31] = 0x964, + [MIB_RSCR33] = 0x96c, + [MIB_RSCR35] = 0x974, + [MIB_RSCR36] = 0x978, + [MIB_BSCR0] = 0x9cc, + [MIB_BSCR1] = 0x9d0, + [MIB_BSCR2] = 0x9d4, + [MIB_BSCR3] = 0x9d8, + [MIB_BSCR4] = 0x9dc, + [MIB_BSCR5] = 0x9e0, + [MIB_BSCR6] = 0x9e4, + [MIB_BSCR7] = 0x9e8, + [MIB_BSCR17] = 0xa10, + [MIB_TRDR1] = 0xa28, +}; + +static const u32 mt7992_offs[] = { + [MIB_RVSR0] = 0x760, + [MIB_RVSR1] = 0x764, + [MIB_BTSCR5] = 0x7c8, + [MIB_BTSCR6] = 0x7d8, + [MIB_RSCR1] = 0x7f0, + [MIB_RSCR27] = 0x998, + [MIB_RSCR28] = 0x99c, + [MIB_RSCR29] = 0x9a0, + [MIB_RSCR30] = 0x9a4, + [MIB_RSCR31] = 0x9a8, + [MIB_RSCR33] = 0x9b0, + [MIB_RSCR35] = 0x9b8, + [MIB_RSCR36] = 0x9bc, + [MIB_BSCR0] = 0xac8, + [MIB_BSCR1] = 0xacc, + [MIB_BSCR2] = 0xad0, + [MIB_BSCR3] = 0xad4, + [MIB_BSCR4] = 0xad8, + [MIB_BSCR5] = 0xadc, + [MIB_BSCR6] = 0xae0, + [MIB_BSCR7] = 0xae4, + [MIB_BSCR17] = 0xb0c, + [MIB_TRDR1] = 0xb24, +}; + static const struct __map mt7996_reg_map[] = { { 0x54000000, 0x02000, 0x1000 }, /* WFDMA_0 (PCIE0 MCU DMA0) */ { 0x55000000, 0x03000, 0x1000 }, /* WFDMA_1 (PCIE0 MCU DMA1) */ @@ -191,6 +249,169 @@ static u32 mt7996_rmw(struct mt76_dev *mdev, u32 offset, u32 mask, u32 val) return dev->bus_ops->rmw(mdev, __mt7996_reg_addr(dev, offset), mask, val); } +#ifdef CONFIG_NET_MEDIATEK_SOC_WED +static int mt7996_mmio_wed_reset(struct mtk_wed_device *wed) +{ + struct mt76_dev *mdev = container_of(wed, struct mt76_dev, mmio.wed); + struct mt7996_dev *dev = container_of(mdev, struct mt7996_dev, mt76); + struct mt76_phy *mphy = &dev->mphy; + int ret; + + ASSERT_RTNL(); + + if (test_and_set_bit(MT76_STATE_WED_RESET, &mphy->state)) + return -EBUSY; + + ret = mt7996_mcu_set_ser(dev, UNI_CMD_SER_TRIGGER, UNI_CMD_SER_SET_RECOVER_L1, + mphy->band_idx); + if (ret) + goto out; + + rtnl_unlock(); + if (!wait_for_completion_timeout(&mdev->mmio.wed_reset, 20 * HZ)) { + dev_err(mdev->dev, "wed reset timeout\n"); + ret = -ETIMEDOUT; + } + rtnl_lock(); +out: + clear_bit(MT76_STATE_WED_RESET, &mphy->state); + + return ret; +} +#endif + +int mt7996_mmio_wed_init(struct mt7996_dev *dev, void *pdev_ptr, + bool hif2, int *irq) +{ +#ifdef CONFIG_NET_MEDIATEK_SOC_WED + struct mtk_wed_device *wed = &dev->mt76.mmio.wed; + struct pci_dev *pci_dev = pdev_ptr; + u32 hif1_ofs = 0; + + if (!wed_enable) + return 0; + + dev->has_rro = true; + + hif1_ofs = MT_WFDMA0_PCIE1(0) - MT_WFDMA0(0); + + if (hif2) + wed = &dev->mt76.mmio.wed_hif2; + + wed->wlan.pci_dev = pci_dev; + wed->wlan.bus_type = MTK_WED_BUS_PCIE; + + wed->wlan.base = devm_ioremap(dev->mt76.dev, + pci_resource_start(pci_dev, 0), + pci_resource_len(pci_dev, 0)); + wed->wlan.phy_base = pci_resource_start(pci_dev, 0); + + if (hif2) { + wed->wlan.wpdma_int = wed->wlan.phy_base + + MT_INT_PCIE1_SOURCE_CSR_EXT; + wed->wlan.wpdma_mask = wed->wlan.phy_base + + MT_INT_PCIE1_MASK_CSR; + wed->wlan.wpdma_tx = wed->wlan.phy_base + hif1_ofs + + MT_TXQ_RING_BASE(0) + + MT7996_TXQ_BAND2 * MT_RING_SIZE; + if (dev->has_rro) { + wed->wlan.wpdma_txfree = wed->wlan.phy_base + hif1_ofs + + MT_RXQ_RING_BASE(0) + + MT7996_RXQ_TXFREE2 * MT_RING_SIZE; + wed->wlan.txfree_tbit = ffs(MT_INT_RX_TXFREE_EXT) - 1; + } else { + wed->wlan.wpdma_txfree = wed->wlan.phy_base + hif1_ofs + + MT_RXQ_RING_BASE(0) + + MT7996_RXQ_MCU_WA_TRI * MT_RING_SIZE; + wed->wlan.txfree_tbit = ffs(MT_INT_RX_DONE_WA_TRI) - 1; + } + + wed->wlan.wpdma_rx_glo = wed->wlan.phy_base + hif1_ofs + MT_WFDMA0_GLO_CFG; + wed->wlan.wpdma_rx = wed->wlan.phy_base + hif1_ofs + + MT_RXQ_RING_BASE(MT7996_RXQ_BAND0) + + MT7996_RXQ_BAND0 * MT_RING_SIZE; + + wed->wlan.id = 0x7991; + wed->wlan.tx_tbit[0] = ffs(MT_INT_TX_DONE_BAND2) - 1; + } else { + wed->wlan.hw_rro = dev->has_rro; /* default on */ + wed->wlan.wpdma_int = wed->wlan.phy_base + MT_INT_SOURCE_CSR; + wed->wlan.wpdma_mask = wed->wlan.phy_base + MT_INT_MASK_CSR; + wed->wlan.wpdma_tx = wed->wlan.phy_base + MT_TXQ_RING_BASE(0) + + MT7996_TXQ_BAND0 * MT_RING_SIZE; + + wed->wlan.wpdma_rx_glo = wed->wlan.phy_base + MT_WFDMA0_GLO_CFG; + + wed->wlan.wpdma_rx = wed->wlan.phy_base + + MT_RXQ_RING_BASE(MT7996_RXQ_BAND0) + + MT7996_RXQ_BAND0 * MT_RING_SIZE; + + wed->wlan.wpdma_rx_rro[0] = wed->wlan.phy_base + + MT_RXQ_RING_BASE(MT7996_RXQ_RRO_BAND0) + + MT7996_RXQ_RRO_BAND0 * MT_RING_SIZE; + wed->wlan.wpdma_rx_rro[1] = wed->wlan.phy_base + hif1_ofs + + MT_RXQ_RING_BASE(MT7996_RXQ_RRO_BAND2) + + MT7996_RXQ_RRO_BAND2 * MT_RING_SIZE; + wed->wlan.wpdma_rx_pg = wed->wlan.phy_base + + MT_RXQ_RING_BASE(MT7996_RXQ_MSDU_PG_BAND0) + + MT7996_RXQ_MSDU_PG_BAND0 * MT_RING_SIZE; + + wed->wlan.rx_nbuf = 65536; + wed->wlan.rx_npkt = dev->hif2 ? 32768 : 24576; + wed->wlan.rx_size = SKB_WITH_OVERHEAD(MT_RX_BUF_SIZE); + + wed->wlan.rx_tbit[0] = ffs(MT_INT_RX_DONE_BAND0) - 1; + wed->wlan.rx_tbit[1] = ffs(MT_INT_RX_DONE_BAND2) - 1; + + wed->wlan.rro_rx_tbit[0] = ffs(MT_INT_RX_DONE_RRO_BAND0) - 1; + wed->wlan.rro_rx_tbit[1] = ffs(MT_INT_RX_DONE_RRO_BAND2) - 1; + + wed->wlan.rx_pg_tbit[0] = ffs(MT_INT_RX_DONE_MSDU_PG_BAND0) - 1; + wed->wlan.rx_pg_tbit[1] = ffs(MT_INT_RX_DONE_MSDU_PG_BAND1) - 1; + wed->wlan.rx_pg_tbit[2] = ffs(MT_INT_RX_DONE_MSDU_PG_BAND2) - 1; + + wed->wlan.tx_tbit[0] = ffs(MT_INT_TX_DONE_BAND0) - 1; + wed->wlan.tx_tbit[1] = ffs(MT_INT_TX_DONE_BAND1) - 1; + if (dev->has_rro) { + wed->wlan.wpdma_txfree = wed->wlan.phy_base + MT_RXQ_RING_BASE(0) + + MT7996_RXQ_TXFREE0 * MT_RING_SIZE; + wed->wlan.txfree_tbit = ffs(MT_INT_RX_TXFREE_MAIN) - 1; + } else { + wed->wlan.txfree_tbit = ffs(MT_INT_RX_DONE_WA_MAIN) - 1; + wed->wlan.wpdma_txfree = wed->wlan.phy_base + MT_RXQ_RING_BASE(0) + + MT7996_RXQ_MCU_WA_MAIN * MT_RING_SIZE; + } + dev->mt76.rx_token_size = MT7996_TOKEN_SIZE + wed->wlan.rx_npkt; + } + + wed->wlan.nbuf = MT7996_HW_TOKEN_SIZE; + wed->wlan.token_start = MT7996_TOKEN_SIZE - wed->wlan.nbuf; + + wed->wlan.amsdu_max_subframes = 8; + wed->wlan.amsdu_max_len = 1536; + + wed->wlan.init_buf = mt7996_wed_init_buf; + wed->wlan.init_rx_buf = mt76_mmio_wed_init_rx_buf; + wed->wlan.release_rx_buf = mt76_mmio_wed_release_rx_buf; + wed->wlan.offload_enable = mt76_mmio_wed_offload_enable; + wed->wlan.offload_disable = mt76_mmio_wed_offload_disable; + if (!hif2) { + wed->wlan.reset = mt7996_mmio_wed_reset; + wed->wlan.reset_complete = mt76_mmio_wed_reset_complete; + } + + if (mtk_wed_device_attach(wed)) + return 0; + + *irq = wed->irq; + dev->mt76.dma_dev = wed->dev; + + return 1; +#else + return 0; +#endif +} + static int mt7996_mmio_init(struct mt76_dev *mdev, void __iomem *mem_base, u32 device_id) @@ -204,6 +425,13 @@ static int mt7996_mmio_init(struct mt76_dev *mdev, switch (device_id) { case 0x7990: dev->reg.base = mt7996_reg_base; + dev->reg.offs_rev = mt7996_offs; + dev->reg.map = mt7996_reg_map; + dev->reg.map_size = ARRAY_SIZE(mt7996_reg_map); + break; + case 0x7992: + dev->reg.base = mt7996_reg_base; + dev->reg.offs_rev = mt7992_offs; dev->reg.map = mt7996_reg_map; dev->reg.map_size = ARRAY_SIZE(mt7996_reg_map); break; @@ -241,8 +469,17 @@ void mt7996_dual_hif_set_irq_mask(struct mt7996_dev *dev, bool write_reg, mdev->mmio.irqmask |= set; if (write_reg) { - mt76_wr(dev, MT_INT_MASK_CSR, mdev->mmio.irqmask); - mt76_wr(dev, MT_INT1_MASK_CSR, mdev->mmio.irqmask); + if (mtk_wed_device_active(&mdev->mmio.wed)) { + mtk_wed_device_irq_set_mask(&mdev->mmio.wed, + mdev->mmio.irqmask); + if (mtk_wed_device_active(&mdev->mmio.wed_hif2)) { + mtk_wed_device_irq_set_mask(&mdev->mmio.wed_hif2, + mdev->mmio.irqmask); + } + } else { + mt76_wr(dev, MT_INT_MASK_CSR, mdev->mmio.irqmask); + mt76_wr(dev, MT_INT1_MASK_CSR, mdev->mmio.irqmask); + } } spin_unlock_irqrestore(&mdev->mmio.irq_lock, flags); @@ -260,22 +497,36 @@ static void mt7996_rx_poll_complete(struct mt76_dev *mdev, static void mt7996_irq_tasklet(struct tasklet_struct *t) { struct mt7996_dev *dev = from_tasklet(dev, t, mt76.irq_tasklet); + struct mtk_wed_device *wed = &dev->mt76.mmio.wed; + struct mtk_wed_device *wed_hif2 = &dev->mt76.mmio.wed_hif2; u32 i, intr, mask, intr1; - mt76_wr(dev, MT_INT_MASK_CSR, 0); - if (dev->hif2) - mt76_wr(dev, MT_INT1_MASK_CSR, 0); - - intr = mt76_rr(dev, MT_INT_SOURCE_CSR); - intr &= dev->mt76.mmio.irqmask; - mt76_wr(dev, MT_INT_SOURCE_CSR, intr); - - if (dev->hif2) { - intr1 = mt76_rr(dev, MT_INT1_SOURCE_CSR); - intr1 &= dev->mt76.mmio.irqmask; - mt76_wr(dev, MT_INT1_SOURCE_CSR, intr1); + if (dev->hif2 && mtk_wed_device_active(wed_hif2)) { + mtk_wed_device_irq_set_mask(wed_hif2, 0); + intr1 = mtk_wed_device_irq_get(wed_hif2, + dev->mt76.mmio.irqmask); + if (intr1 & MT_INT_RX_TXFREE_EXT) + napi_schedule(&dev->mt76.napi[MT_RXQ_TXFREE_BAND2]); + } - intr |= intr1; + if (mtk_wed_device_active(wed)) { + mtk_wed_device_irq_set_mask(wed, 0); + intr = mtk_wed_device_irq_get(wed, dev->mt76.mmio.irqmask); + intr |= (intr1 & ~MT_INT_RX_TXFREE_EXT); + } else { + mt76_wr(dev, MT_INT_MASK_CSR, 0); + if (dev->hif2) + mt76_wr(dev, MT_INT1_MASK_CSR, 0); + + intr = mt76_rr(dev, MT_INT_SOURCE_CSR); + intr &= dev->mt76.mmio.irqmask; + mt76_wr(dev, MT_INT_SOURCE_CSR, intr); + if (dev->hif2) { + intr1 = mt76_rr(dev, MT_INT1_SOURCE_CSR); + intr1 &= dev->mt76.mmio.irqmask; + mt76_wr(dev, MT_INT1_SOURCE_CSR, intr1); + intr |= intr1; + } } trace_dev_irq(&dev->mt76, intr, dev->mt76.mmio.irqmask); @@ -308,9 +559,17 @@ irqreturn_t mt7996_irq_handler(int irq, void *dev_instance) { struct mt7996_dev *dev = dev_instance; - mt76_wr(dev, MT_INT_MASK_CSR, 0); - if (dev->hif2) - mt76_wr(dev, MT_INT1_MASK_CSR, 0); + if (mtk_wed_device_active(&dev->mt76.mmio.wed)) + mtk_wed_device_irq_set_mask(&dev->mt76.mmio.wed, 0); + else + mt76_wr(dev, MT_INT_MASK_CSR, 0); + + if (dev->hif2) { + if (mtk_wed_device_active(&dev->mt76.mmio.wed_hif2)) + mtk_wed_device_irq_set_mask(&dev->mt76.mmio.wed_hif2, 0); + else + mt76_wr(dev, MT_INT1_MASK_CSR, 0); + } if (!test_bit(MT76_STATE_INITIALIZED, &dev->mphy.state)) return IRQ_NONE; diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h b/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h index e53cf6a3704c4a..bc73bcb47bf02c 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h @@ -13,6 +13,7 @@ #define MT7996_MAX_INTERFACES 19 /* per-band */ #define MT7996_MAX_WMM_SETS 4 +#define MT7996_WTBL_BMC_SIZE (is_mt7992(&dev->mt76) ? 32 : 64) #define MT7996_WTBL_RESERVED (mt7996_wtbl_size(dev) - 1) #define MT7996_WTBL_STA (MT7996_WTBL_RESERVED - \ mt7996_max_interface_num(dev)) @@ -33,22 +34,54 @@ #define MT7996_FIRMWARE_DSP "mediatek/mt7996/mt7996_dsp.bin" #define MT7996_ROM_PATCH "mediatek/mt7996/mt7996_rom_patch.bin" +#define MT7992_FIRMWARE_WA "mediatek/mt7996/mt7992_wa.bin" +#define MT7992_FIRMWARE_WM "mediatek/mt7996/mt7992_wm.bin" +#define MT7992_FIRMWARE_DSP "mediatek/mt7996/mt7992_dsp.bin" +#define MT7992_ROM_PATCH "mediatek/mt7996/mt7992_rom_patch.bin" + #define MT7996_EEPROM_DEFAULT "mediatek/mt7996/mt7996_eeprom.bin" +#define MT7992_EEPROM_DEFAULT "mediatek/mt7996/mt7992_eeprom.bin" #define MT7996_EEPROM_SIZE 7680 #define MT7996_EEPROM_BLOCK_SIZE 16 #define MT7996_TOKEN_SIZE 16384 +#define MT7996_HW_TOKEN_SIZE 8192 #define MT7996_CFEND_RATE_DEFAULT 0x49 /* OFDM 24M */ #define MT7996_CFEND_RATE_11B 0x03 /* 11B LP, 11M */ +#define MT7996_SKU_RATE_NUM 417 + #define MT7996_MAX_TWT_AGRT 16 #define MT7996_MAX_STA_TWT_AGRT 8 #define MT7996_MAX_QUEUE (__MT_RXQ_MAX + __MT_MCUQ_MAX + 3) /* NOTE: used to map mt76_rates. idx may change if firmware expands table */ -#define MT7996_BASIC_RATES_TBL 11 +#define MT7996_BASIC_RATES_TBL 31 #define MT7996_BEACON_RATES_TBL 25 +#define MT7996_THERMAL_THROTTLE_MAX 100 +#define MT7996_CDEV_THROTTLE_MAX 99 +#define MT7996_CRIT_TEMP_IDX 0 +#define MT7996_MAX_TEMP_IDX 1 +#define MT7996_CRIT_TEMP 110 +#define MT7996_MAX_TEMP 120 + +#define MT7996_RRO_MAX_SESSION 1024 +#define MT7996_RRO_WINDOW_MAX_LEN 1024 +#define MT7996_RRO_ADDR_ELEM_LEN 128 +#define MT7996_RRO_BA_BITMAP_LEN 2 +#define MT7996_RRO_BA_BITMAP_CR_SIZE ((MT7996_RRO_MAX_SESSION * 128) / \ + MT7996_RRO_BA_BITMAP_LEN) +#define MT7996_RRO_BA_BITMAP_SESSION_SIZE (MT7996_RRO_MAX_SESSION / \ + MT7996_RRO_ADDR_ELEM_LEN) +#define MT7996_RRO_WINDOW_MAX_SIZE (MT7996_RRO_WINDOW_MAX_LEN * \ + MT7996_RRO_BA_BITMAP_SESSION_SIZE) + +#define MT7996_RX_BUF_SIZE (1800 + \ + SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) +#define MT7996_RX_MSDU_PAGE_SIZE (128 + \ + SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) + struct mt7996_vif; struct mt7996_sta; struct mt7996_dfs_pulse; @@ -73,11 +106,21 @@ enum mt7996_rxq_id { MT7996_RXQ_MCU_WM = 0, MT7996_RXQ_MCU_WA, MT7996_RXQ_MCU_WA_MAIN = 2, - MT7996_RXQ_MCU_WA_EXT = 2,/* unused */ + MT7996_RXQ_MCU_WA_EXT = 3, /* for mt7992 */ MT7996_RXQ_MCU_WA_TRI = 3, MT7996_RXQ_BAND0 = 4, - MT7996_RXQ_BAND1 = 4,/* unused */ + MT7996_RXQ_BAND1 = 5, /* for mt7992 */ MT7996_RXQ_BAND2 = 5, + MT7996_RXQ_RRO_BAND0 = 8, + MT7996_RXQ_RRO_BAND1 = 8,/* unused */ + MT7996_RXQ_RRO_BAND2 = 6, + MT7996_RXQ_MSDU_PG_BAND0 = 10, + MT7996_RXQ_MSDU_PG_BAND1 = 11, + MT7996_RXQ_MSDU_PG_BAND2 = 12, + MT7996_RXQ_TXFREE0 = 9, + MT7996_RXQ_TXFREE1 = 9, + MT7996_RXQ_TXFREE2 = 7, + MT7996_RXQ_RRO_IND = 0, }; struct mt7996_twt_flow { @@ -146,6 +189,20 @@ struct mt7996_hif { int irq; }; +struct mt7996_wed_rro_addr { + u32 head_low; + u32 head_high : 4; + u32 count: 11; + u32 oor: 1; + u32 rsv : 8; + u32 signature : 8; +}; + +struct mt7996_wed_rro_session_id { + struct list_head list; + u16 id; +}; + struct mt7996_phy { struct mt76_phy *mt76; struct mt7996_dev *dev; @@ -154,6 +211,11 @@ struct mt7996_phy { struct ieee80211_vif *monitor_vif; + struct thermal_cooling_device *cdev; + u8 cdev_state; + u8 throttle_state; + u32 throttle_temp[2]; /* 0: critical high, 1: maximum */ + u32 rxfilter; u64 omac_mask; @@ -164,11 +226,15 @@ struct mt7996_phy { u8 rdd_state; + u16 beacon_rate; + u32 rx_ampdu_ts; u32 ampdu_ref; struct mt76_mib_stats mib; struct mt76_channel_state state_ts; + + bool has_aux_rx; }; struct mt7996_dev { @@ -221,10 +287,28 @@ struct mt7996_dev { u32 hw_pattern; - bool dbdc_support:1; - bool tbtc_support:1; bool flash_mode:1; bool has_eht:1; + bool has_rro:1; + + struct { + struct { + void *ptr; + dma_addr_t phy_addr; + } ba_bitmap[MT7996_RRO_BA_BITMAP_LEN]; + struct { + void *ptr; + dma_addr_t phy_addr; + } addr_elem[MT7996_RRO_ADDR_ELEM_LEN]; + struct { + void *ptr; + dma_addr_t phy_addr; + } session; + + struct work_struct work; + struct list_head poll_list; + spinlock_t lock; + } wed_rro; bool ibf; u8 fw_debug_wm; @@ -314,6 +398,20 @@ mt7996_phy3(struct mt7996_dev *dev) return __mt7996_phy(dev, MT_BAND2); } +static inline bool +mt7996_band_valid(struct mt7996_dev *dev, u8 band) +{ + if (is_mt7992(&dev->mt76)) + return band <= MT_BAND1; + + /* tri-band support */ + if (band <= MT_BAND2 && + mt76_get_field(dev, MT_PAD_GPIO, MT_PAD_GPIO_ADIE_COMB) <= 1) + return true; + + return band == MT_BAND0 || band == MT_BAND2; +} + extern const struct ieee80211_ops mt7996_ops; extern struct pci_driver mt7996_pci_driver; extern struct pci_driver mt7996_hif_driver; @@ -334,9 +432,10 @@ int mt7996_dma_init(struct mt7996_dev *dev); void mt7996_dma_reset(struct mt7996_dev *dev, bool force); void mt7996_dma_prefetch(struct mt7996_dev *dev); void mt7996_dma_cleanup(struct mt7996_dev *dev); -void mt7996_dma_start(struct mt7996_dev *dev, bool reset); -void mt7996_init_txpower(struct mt7996_dev *dev, - struct ieee80211_supported_band *sband); +void mt7996_dma_start(struct mt7996_dev *dev, bool reset, bool wed_reset); +int mt7996_init_tx_queues(struct mt7996_phy *phy, int idx, + int n_desc, int ring_base, struct mtk_wed_device *wed); +void mt7996_init_txpower(struct mt7996_phy *phy); int mt7996_txbf_init(struct mt7996_dev *dev); void mt7996_reset(struct mt7996_dev *dev); int mt7996_run(struct ieee80211_hw *hw); @@ -373,6 +472,8 @@ int mt7996_mcu_set_chan_info(struct mt7996_phy *phy, u16 tag); int mt7996_mcu_set_tx(struct mt7996_dev *dev, struct ieee80211_vif *vif); int mt7996_mcu_set_fixed_rate_ctrl(struct mt7996_dev *dev, void *data, u16 version); +int mt7996_mcu_set_fixed_field(struct mt7996_dev *dev, struct ieee80211_vif *vif, + struct ieee80211_sta *sta, void *data, u32 field); int mt7996_mcu_set_eeprom(struct mt7996_dev *dev); int mt7996_mcu_get_eeprom(struct mt7996_dev *dev, u32 offset); int mt7996_mcu_get_eeprom_free_block(struct mt7996_dev *dev, u8 *block_num); @@ -388,13 +489,19 @@ int mt7996_mcu_set_radio_en(struct mt7996_phy *phy, bool enable); int mt7996_mcu_set_rts_thresh(struct mt7996_phy *phy, u32 val); int mt7996_mcu_set_timing(struct mt7996_phy *phy, struct ieee80211_vif *vif); int mt7996_mcu_get_chan_mib_info(struct mt7996_phy *phy, bool chan_switch); +int mt7996_mcu_get_temperature(struct mt7996_phy *phy); +int mt7996_mcu_set_thermal_throttling(struct mt7996_phy *phy, u8 state); +int mt7996_mcu_set_thermal_protect(struct mt7996_phy *phy, bool enable); +int mt7996_mcu_set_txpower_sku(struct mt7996_phy *phy); int mt7996_mcu_rdd_cmd(struct mt7996_dev *dev, int cmd, u8 index, u8 rx_sel, u8 val); int mt7996_mcu_rdd_background_enable(struct mt7996_phy *phy, struct cfg80211_chan_def *chandef); +int mt7996_mcu_set_fixed_rate_table(struct mt7996_phy *phy, u8 table_idx, + u16 rate_idx, bool beacon); int mt7996_mcu_rf_regval(struct mt7996_dev *dev, u32 regidx, u32 *val, bool set); int mt7996_mcu_set_hdr_trans(struct mt7996_dev *dev, bool hdr_trans); -int mt7996_mcu_set_rro(struct mt7996_dev *dev, u16 tag, u8 val); +int mt7996_mcu_set_rro(struct mt7996_dev *dev, u16 tag, u16 val); int mt7996_mcu_wa_cmd(struct mt7996_dev *dev, int cmd, u32 a1, u32 a2, u32 a3); int mt7996_mcu_fw_log_2_host(struct mt7996_dev *dev, u8 type, u8 ctrl); int mt7996_mcu_fw_dbg_ctrl(struct mt7996_dev *dev, u32 module, u8 level); @@ -402,15 +509,18 @@ int mt7996_mcu_trigger_assert(struct mt7996_dev *dev); void mt7996_mcu_rx_event(struct mt7996_dev *dev, struct sk_buff *skb); void mt7996_mcu_exit(struct mt7996_dev *dev); int mt7996_mcu_get_all_sta_info(struct mt7996_phy *phy, u16 tag); +int mt7996_mcu_wed_rro_reset_sessions(struct mt7996_dev *dev, u16 id); static inline u8 mt7996_max_interface_num(struct mt7996_dev *dev) { - return MT7996_MAX_INTERFACES * (1 + dev->dbdc_support + dev->tbtc_support); + return min(MT7996_MAX_INTERFACES * (1 + mt7996_band_valid(dev, MT_BAND1) + + mt7996_band_valid(dev, MT_BAND2)), + MT7996_WTBL_BMC_SIZE); } static inline u16 mt7996_wtbl_size(struct mt7996_dev *dev) { - return (dev->wtbl_size_group << 8) + 64; + return (dev->wtbl_size_group << 8) + MT7996_WTBL_BMC_SIZE; } void mt7996_dual_hif_set_irq_mask(struct mt7996_dev *dev, bool write_reg, @@ -437,6 +547,18 @@ static inline void mt7996_irq_disable(struct mt7996_dev *dev, u32 mask) void mt7996_memcpy_fromio(struct mt7996_dev *dev, void *buf, u32 offset, size_t len); +static inline u16 mt7996_rx_chainmask(struct mt7996_phy *phy) +{ + int max_nss = hweight8(phy->mt76->hw->wiphy->available_antennas_tx); + int cur_nss = hweight8(phy->mt76->antenna_mask); + u16 tx_chainmask = phy->mt76->chainmask; + + if (cur_nss != max_nss) + return tx_chainmask; + + return tx_chainmask | (BIT(fls(tx_chainmask)) * phy->has_aux_rx); +} + void mt7996_mac_init(struct mt7996_dev *dev); u32 mt7996_mac_wtbl_lmac_addr(struct mt7996_dev *dev, u16 wcid, u8 dw); bool mt7996_mac_wtbl_update(struct mt7996_dev *dev, int idx, u32 mask); @@ -445,8 +567,6 @@ void mt7996_mac_cca_stats_reset(struct mt7996_phy *phy); void mt7996_mac_enable_nf(struct mt7996_dev *dev, u8 band); void mt7996_mac_enable_rtscts(struct mt7996_dev *dev, struct ieee80211_vif *vif, bool enable); -void mt7996_mac_set_fixed_rate_table(struct mt7996_dev *dev, - u8 tbl_idx, u16 rate_idx); void mt7996_mac_write_txwi(struct mt7996_dev *dev, __le32 *txwi, struct sk_buff *skb, struct mt76_wcid *wcid, struct ieee80211_key_conf *key, int pid, @@ -485,9 +605,10 @@ int mt7996_init_debugfs(struct mt7996_phy *phy); void mt7996_debugfs_rx_fw_monitor(struct mt7996_dev *dev, const void *data, int len); bool mt7996_debugfs_rx_log(struct mt7996_dev *dev, const void *data, int len); int mt7996_mcu_add_key(struct mt76_dev *dev, struct ieee80211_vif *vif, - struct mt76_connac_sta_key_conf *sta_key_conf, struct ieee80211_key_conf *key, int mcu_cmd, struct mt76_wcid *wcid, enum set_key_cmd cmd); +int mt7996_mcu_bcn_prot_enable(struct mt7996_dev *dev, struct ieee80211_vif *vif, + struct ieee80211_key_conf *key); int mt7996_mcu_wtbl_update_hdr_trans(struct mt7996_dev *dev, struct ieee80211_vif *vif, struct ieee80211_sta *sta); @@ -495,5 +616,16 @@ int mt7996_mcu_wtbl_update_hdr_trans(struct mt7996_dev *dev, void mt7996_sta_add_debugfs(struct ieee80211_hw *hw, struct ieee80211_vif *vif, struct ieee80211_sta *sta, struct dentry *dir); #endif +int mt7996_mmio_wed_init(struct mt7996_dev *dev, void *pdev_ptr, + bool hif2, int *irq); +u32 mt7996_wed_init_buf(void *ptr, dma_addr_t phys, int token_id); + +#ifdef CONFIG_MTK_DEBUG +int mt7996_mtk_init_debugfs(struct mt7996_phy *phy, struct dentry *dir); +#endif + +#ifdef CONFIG_NET_MEDIATEK_SOC_WED +int mt7996_dma_rro_init(struct mt7996_dev *dev); +#endif /* CONFIG_NET_MEDIATEK_SOC_WED */ #endif diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/pci.c b/drivers/net/wireless/mediatek/mt76/mt7996/pci.c index c5301050ff8b30..04056181368a69 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/pci.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/pci.c @@ -17,11 +17,13 @@ static u32 hif_idx; static const struct pci_device_id mt7996_pci_device_table[] = { { PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, 0x7990) }, + { PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, 0x7992) }, { }, }; static const struct pci_device_id mt7996_hif_device_table[] = { { PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, 0x7991) }, + { PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, 0x799a) }, { }, }; @@ -60,7 +62,9 @@ static void mt7996_put_hif2(struct mt7996_hif *hif) static struct mt7996_hif *mt7996_pci_init_hif2(struct pci_dev *pdev) { hif_idx++; - if (!pci_get_device(PCI_VENDOR_ID_MEDIATEK, 0x7991, NULL)) + + if (!pci_get_device(PCI_VENDOR_ID_MEDIATEK, 0x7991, NULL) && + !pci_get_device(PCI_VENDOR_ID_MEDIATEK, 0x799a, NULL)) return NULL; writel(hif_idx | MT_PCIE_RECOG_ID_SEM, @@ -92,10 +96,10 @@ static int mt7996_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) { struct pci_dev *hif2_dev; + struct mt7996_hif *hif2; struct mt7996_dev *dev; + int irq, hif2_irq, ret; struct mt76_dev *mdev; - struct mt7996_hif *hif2; - int irq, ret; ret = pcim_enable_device(pdev); if (ret) @@ -107,13 +111,17 @@ static int mt7996_pci_probe(struct pci_dev *pdev, pci_set_master(pdev); - ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); + ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(36)); + if (ret) + return ret; + + ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32)); if (ret) return ret; mt76_pci_disable_aspm(pdev); - if (id->device == 0x7991) + if (id->device == 0x7991 || id->device == 0x799a) return mt7996_pci_hif2_probe(pdev); dev = mt7996_mmio_probe(&pdev->dev, pcim_iomap_table(pdev)[0], @@ -125,15 +133,22 @@ static int mt7996_pci_probe(struct pci_dev *pdev, mt7996_wfsys_reset(dev); hif2 = mt7996_pci_init_hif2(pdev); - ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES); + ret = mt7996_mmio_wed_init(dev, pdev, false, &irq); if (ret < 0) - goto free_device; + goto free_wed_or_irq_vector; + + if (!ret) { + ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES); + if (ret < 0) + goto free_device; + + irq = pdev->irq; + } - irq = pdev->irq; ret = devm_request_irq(mdev->dev, irq, mt7996_irq_handler, IRQF_SHARED, KBUILD_MODNAME, dev); if (ret) - goto free_irq_vector; + goto free_wed_or_irq_vector; mt76_wr(dev, MT_INT_MASK_CSR, 0); /* master switch of PCIe tnterrupt enable */ @@ -143,16 +158,25 @@ static int mt7996_pci_probe(struct pci_dev *pdev, hif2_dev = container_of(hif2->dev, struct pci_dev, dev); dev->hif2 = hif2; - ret = pci_alloc_irq_vectors(hif2_dev, 1, 1, PCI_IRQ_ALL_TYPES); + ret = mt7996_mmio_wed_init(dev, hif2_dev, true, &hif2_irq); if (ret < 0) - goto free_hif2; + goto free_hif2_wed_irq_vector; + + if (!ret) { + ret = pci_alloc_irq_vectors(hif2_dev, 1, 1, + PCI_IRQ_ALL_TYPES); + if (ret < 0) + goto free_hif2; - dev->hif2->irq = hif2_dev->irq; - ret = devm_request_irq(mdev->dev, dev->hif2->irq, - mt7996_irq_handler, IRQF_SHARED, - KBUILD_MODNAME "-hif", dev); + dev->hif2->irq = hif2_dev->irq; + hif2_irq = dev->hif2->irq; + } + + ret = devm_request_irq(mdev->dev, hif2_irq, mt7996_irq_handler, + IRQF_SHARED, KBUILD_MODNAME "-hif", + dev); if (ret) - goto free_hif2_irq_vector; + goto free_hif2_wed_irq_vector; mt76_wr(dev, MT_INT1_MASK_CSR, 0); /* master switch of PCIe tnterrupt enable */ @@ -167,16 +191,23 @@ static int mt7996_pci_probe(struct pci_dev *pdev, free_hif2_irq: if (dev->hif2) - devm_free_irq(mdev->dev, dev->hif2->irq, dev); -free_hif2_irq_vector: - if (dev->hif2) - pci_free_irq_vectors(hif2_dev); + devm_free_irq(mdev->dev, hif2_irq, dev); +free_hif2_wed_irq_vector: + if (dev->hif2) { + if (mtk_wed_device_active(&dev->mt76.mmio.wed_hif2)) + mtk_wed_device_detach(&dev->mt76.mmio.wed_hif2); + else + pci_free_irq_vectors(hif2_dev); + } free_hif2: if (dev->hif2) put_device(dev->hif2->dev); devm_free_irq(mdev->dev, irq, dev); -free_irq_vector: - pci_free_irq_vectors(pdev); +free_wed_or_irq_vector: + if (mtk_wed_device_active(&dev->mt76.mmio.wed)) + mtk_wed_device_detach(&dev->mt76.mmio.wed); + else + pci_free_irq_vectors(pdev); free_device: mt76_free_device(&dev->mt76); @@ -221,3 +252,7 @@ MODULE_FIRMWARE(MT7996_FIRMWARE_WA); MODULE_FIRMWARE(MT7996_FIRMWARE_WM); MODULE_FIRMWARE(MT7996_FIRMWARE_DSP); MODULE_FIRMWARE(MT7996_ROM_PATCH); +MODULE_FIRMWARE(MT7992_FIRMWARE_WA); +MODULE_FIRMWARE(MT7992_FIRMWARE_WM); +MODULE_FIRMWARE(MT7992_FIRMWARE_DSP); +MODULE_FIRMWARE(MT7992_ROM_PATCH); diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/regs.h b/drivers/net/wireless/mediatek/mt76/mt7996/regs.h index 0086a78666579e..47b429d8bfbe5d 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/regs.h +++ b/drivers/net/wireless/mediatek/mt76/mt7996/regs.h @@ -19,6 +19,7 @@ struct __base { /* used to differentiate between generations */ struct mt7996_reg_desc { const struct __base *base; + const u32 *offs_rev; const struct __map *map; u32 map_size; }; @@ -39,6 +40,73 @@ enum base_rev { #define __BASE(_id, _band) (dev->reg.base[(_id)].band_base[(_band)]) +enum offs_rev { + MIB_RVSR0, + MIB_RVSR1, + MIB_BTSCR5, + MIB_BTSCR6, + MIB_RSCR1, + MIB_RSCR27, + MIB_RSCR28, + MIB_RSCR29, + MIB_RSCR30, + MIB_RSCR31, + MIB_RSCR33, + MIB_RSCR35, + MIB_RSCR36, + MIB_BSCR0, + MIB_BSCR1, + MIB_BSCR2, + MIB_BSCR3, + MIB_BSCR4, + MIB_BSCR5, + MIB_BSCR6, + MIB_BSCR7, + MIB_BSCR17, + MIB_TRDR1, + __MT_OFFS_MAX, +}; + +#define __OFFS(id) (dev->reg.offs_rev[(id)]) + +/* RRO TOP */ +#define MT_RRO_TOP_BASE 0xA000 +#define MT_RRO_TOP(ofs) (MT_RRO_TOP_BASE + (ofs)) + +#define MT_RRO_BA_BITMAP_BASE0 MT_RRO_TOP(0x8) +#define MT_RRO_BA_BITMAP_BASE1 MT_RRO_TOP(0xC) +#define WF_RRO_AXI_MST_CFG MT_RRO_TOP(0xB8) +#define WF_RRO_AXI_MST_CFG_DIDX_OK BIT(12) +#define MT_RRO_ADDR_ARRAY_BASE1 MT_RRO_TOP(0x34) +#define MT_RRO_ADDR_ARRAY_ELEM_ADDR_SEG_MODE BIT(31) + +#define MT_RRO_IND_CMD_SIGNATURE_BASE0 MT_RRO_TOP(0x38) +#define MT_RRO_IND_CMD_SIGNATURE_BASE1 MT_RRO_TOP(0x3C) +#define MT_RRO_IND_CMD_0_CTRL0 MT_RRO_TOP(0x40) +#define MT_RRO_IND_CMD_SIGNATURE_BASE1_EN BIT(31) + +#define MT_RRO_PARTICULAR_CFG0 MT_RRO_TOP(0x5C) +#define MT_RRO_PARTICULAR_CFG1 MT_RRO_TOP(0x60) +#define MT_RRO_PARTICULAR_CONFG_EN BIT(31) +#define MT_RRO_PARTICULAR_SID GENMASK(30, 16) + +#define MT_RRO_BA_BITMAP_BASE_EXT0 MT_RRO_TOP(0x70) +#define MT_RRO_BA_BITMAP_BASE_EXT1 MT_RRO_TOP(0x74) +#define MT_RRO_HOST_INT_ENA MT_RRO_TOP(0x204) +#define MT_RRO_HOST_INT_ENA_HOST_RRO_DONE_ENA BIT(0) + +#define MT_RRO_ADDR_ELEM_SEG_ADDR0 MT_RRO_TOP(0x400) + +#define MT_RRO_ACK_SN_CTRL MT_RRO_TOP(0x50) +#define MT_RRO_ACK_SN_CTRL_SN_MASK GENMASK(27, 16) +#define MT_RRO_ACK_SN_CTRL_SESSION_MASK GENMASK(11, 0) + +#define MT_RRO_DBG_RD_CTRL MT_RRO_TOP(0xe0) +#define MT_RRO_DBG_RD_ADDR GENMASK(15, 0) +#define MT_RRO_DBG_RD_EXEC BIT(31) + +#define MT_RRO_DBG_RDAT_DW(_n) MT_RRO_TOP(0xf0 + (_n) * 0x4) + #define MT_MCU_INT_EVENT 0x2108 #define MT_MCU_INT_EVENT_DMA_STOPPED BIT(0) #define MT_MCU_INT_EVENT_DMA_INIT BIT(1) @@ -140,32 +208,32 @@ enum base_rev { #define MT_WF_MIB_BASE(_band) __BASE(WF_MIB_BASE, (_band)) #define MT_WF_MIB(_band, ofs) (MT_WF_MIB_BASE(_band) + (ofs)) -#define MT_MIB_BSCR0(_band) MT_WF_MIB(_band, 0x9cc) -#define MT_MIB_BSCR1(_band) MT_WF_MIB(_band, 0x9d0) -#define MT_MIB_BSCR2(_band) MT_WF_MIB(_band, 0x9d4) -#define MT_MIB_BSCR3(_band) MT_WF_MIB(_band, 0x9d8) -#define MT_MIB_BSCR4(_band) MT_WF_MIB(_band, 0x9dc) -#define MT_MIB_BSCR5(_band) MT_WF_MIB(_band, 0x9e0) -#define MT_MIB_BSCR6(_band) MT_WF_MIB(_band, 0x9e4) -#define MT_MIB_BSCR7(_band) MT_WF_MIB(_band, 0x9e8) -#define MT_MIB_BSCR17(_band) MT_WF_MIB(_band, 0xa10) +#define MT_MIB_BSCR0(_band) MT_WF_MIB(_band, __OFFS(MIB_BSCR0)) +#define MT_MIB_BSCR1(_band) MT_WF_MIB(_band, __OFFS(MIB_BSCR1)) +#define MT_MIB_BSCR2(_band) MT_WF_MIB(_band, __OFFS(MIB_BSCR2)) +#define MT_MIB_BSCR3(_band) MT_WF_MIB(_band, __OFFS(MIB_BSCR3)) +#define MT_MIB_BSCR4(_band) MT_WF_MIB(_band, __OFFS(MIB_BSCR4)) +#define MT_MIB_BSCR5(_band) MT_WF_MIB(_band, __OFFS(MIB_BSCR5)) +#define MT_MIB_BSCR6(_band) MT_WF_MIB(_band, __OFFS(MIB_BSCR6)) +#define MT_MIB_BSCR7(_band) MT_WF_MIB(_band, __OFFS(MIB_BSCR7)) +#define MT_MIB_BSCR17(_band) MT_WF_MIB(_band, __OFFS(MIB_BSCR17)) #define MT_MIB_TSCR5(_band) MT_WF_MIB(_band, 0x6c4) #define MT_MIB_TSCR6(_band) MT_WF_MIB(_band, 0x6c8) #define MT_MIB_TSCR7(_band) MT_WF_MIB(_band, 0x6d0) -#define MT_MIB_RSCR1(_band) MT_WF_MIB(_band, 0x7ac) +#define MT_MIB_RSCR1(_band) MT_WF_MIB(_band, __OFFS(MIB_RSCR1)) /* rx mpdu counter, full 32 bits */ -#define MT_MIB_RSCR31(_band) MT_WF_MIB(_band, 0x964) -#define MT_MIB_RSCR33(_band) MT_WF_MIB(_band, 0x96c) +#define MT_MIB_RSCR31(_band) MT_WF_MIB(_band, __OFFS(MIB_RSCR31)) +#define MT_MIB_RSCR33(_band) MT_WF_MIB(_band, __OFFS(MIB_RSCR33)) #define MT_MIB_SDR6(_band) MT_WF_MIB(_band, 0x020) #define MT_MIB_SDR6_CHANNEL_IDL_CNT_MASK GENMASK(15, 0) -#define MT_MIB_RVSR0(_band) MT_WF_MIB(_band, 0x720) +#define MT_MIB_RVSR0(_band) MT_WF_MIB(_band, __OFFS(MIB_RVSR0)) -#define MT_MIB_RSCR35(_band) MT_WF_MIB(_band, 0x974) -#define MT_MIB_RSCR36(_band) MT_WF_MIB(_band, 0x978) +#define MT_MIB_RSCR35(_band) MT_WF_MIB(_band, __OFFS(MIB_RSCR35)) +#define MT_MIB_RSCR36(_band) MT_WF_MIB(_band, __OFFS(MIB_RSCR36)) /* tx ampdu cnt, full 32 bits */ #define MT_MIB_TSCR0(_band) MT_WF_MIB(_band, 0x6b0) @@ -178,16 +246,16 @@ enum base_rev { #define MT_MIB_TSCR4(_band) MT_WF_MIB(_band, 0x6c0) /* rx ampdu count, 32-bit */ -#define MT_MIB_RSCR27(_band) MT_WF_MIB(_band, 0x954) +#define MT_MIB_RSCR27(_band) MT_WF_MIB(_band, __OFFS(MIB_RSCR27)) /* rx ampdu bytes count, 32-bit */ -#define MT_MIB_RSCR28(_band) MT_WF_MIB(_band, 0x958) +#define MT_MIB_RSCR28(_band) MT_WF_MIB(_band, __OFFS(MIB_RSCR28)) /* rx ampdu valid subframe count */ -#define MT_MIB_RSCR29(_band) MT_WF_MIB(_band, 0x95c) +#define MT_MIB_RSCR29(_band) MT_WF_MIB(_band, __OFFS(MIB_RSCR29)) /* rx ampdu valid subframe bytes count, 32bits */ -#define MT_MIB_RSCR30(_band) MT_WF_MIB(_band, 0x960) +#define MT_MIB_RSCR30(_band) MT_WF_MIB(_band, __OFFS(MIB_RSCR30)) /* remaining windows protected stats */ #define MT_MIB_SDR27(_band) MT_WF_MIB(_band, 0x080) @@ -196,18 +264,18 @@ enum base_rev { #define MT_MIB_SDR28(_band) MT_WF_MIB(_band, 0x084) #define MT_MIB_SDR28_TX_RWP_NEED_CNT GENMASK(15, 0) -#define MT_MIB_RVSR1(_band) MT_WF_MIB(_band, 0x724) +#define MT_MIB_RVSR1(_band) MT_WF_MIB(_band, __OFFS(MIB_RVSR1)) /* rx blockack count, 32 bits */ #define MT_MIB_TSCR1(_band) MT_WF_MIB(_band, 0x6b4) #define MT_MIB_BTSCR0(_band) MT_WF_MIB(_band, 0x5e0) -#define MT_MIB_BTSCR5(_band) MT_WF_MIB(_band, 0x788) -#define MT_MIB_BTSCR6(_band) MT_WF_MIB(_band, 0x798) +#define MT_MIB_BTSCR5(_band) MT_WF_MIB(_band, __OFFS(MIB_BTSCR5)) +#define MT_MIB_BTSCR6(_band) MT_WF_MIB(_band, __OFFS(MIB_BTSCR6)) #define MT_MIB_BFTFCR(_band) MT_WF_MIB(_band, 0x5d0) -#define MT_TX_AGG_CNT(_band, n) MT_WF_MIB(_band, 0xa28 + ((n) << 2)) +#define MT_TX_AGG_CNT(_band, n) MT_WF_MIB(_band, __OFFS(MIB_TRDR1) + ((n) << 2)) #define MT_MIB_ARNG(_band, n) MT_WF_MIB(_band, 0x0b0 + ((n) << 2)) #define MT_MIB_ARNCR_RANGE(val, n) (((val) >> ((n) << 4)) & GENMASK(9, 0)) @@ -330,15 +398,22 @@ enum base_rev { #define MT_WFDMA0_RX_INT_PCIE_SEL MT_WFDMA0(0x154) #define MT_WFDMA0_RX_INT_SEL_RING3 BIT(3) +#define MT_WFDMA0_RX_INT_SEL_RING6 BIT(6) #define MT_WFDMA0_MCU_HOST_INT_ENA MT_WFDMA0(0x1f4) #define MT_WFDMA0_GLO_CFG MT_WFDMA0(0x208) #define MT_WFDMA0_GLO_CFG_TX_DMA_EN BIT(0) #define MT_WFDMA0_GLO_CFG_RX_DMA_EN BIT(2) -#define MT_WFDMA0_GLO_CFG_OMIT_TX_INFO BIT(28) -#define MT_WFDMA0_GLO_CFG_OMIT_RX_INFO BIT(27) #define MT_WFDMA0_GLO_CFG_OMIT_RX_INFO_PFET2 BIT(21) +#define MT_WFDMA0_GLO_CFG_EXT_EN BIT(26) +#define MT_WFDMA0_GLO_CFG_OMIT_RX_INFO BIT(27) +#define MT_WFDMA0_GLO_CFG_OMIT_TX_INFO BIT(28) + +#define MT_WFDMA0_PAUSE_RX_Q_45_TH MT_WFDMA0(0x268) +#define MT_WFDMA0_PAUSE_RX_Q_67_TH MT_WFDMA0(0x26c) +#define MT_WFDMA0_PAUSE_RX_Q_89_TH MT_WFDMA0(0x270) +#define MT_WFDMA0_PAUSE_RX_Q_RRO_TH MT_WFDMA0(0x27c) #define WF_WFDMA0_GLO_CFG_EXT0 MT_WFDMA0(0x2b0) #define WF_WFDMA0_GLO_CFG_EXT0_RX_WB_RXD BIT(18) @@ -362,10 +437,14 @@ enum base_rev { #define MT_WFDMA_HOST_CONFIG MT_WFDMA_EXT_CSR(0x30) #define MT_WFDMA_HOST_CONFIG_PDMA_BAND BIT(0) +#define MT_WFDMA_HOST_CONFIG_BAND2_PCIE1 BIT(22) #define MT_WFDMA_EXT_CSR_HIF_MISC MT_WFDMA_EXT_CSR(0x44) #define MT_WFDMA_EXT_CSR_HIF_MISC_BUSY BIT(0) +#define MT_WFDMA_AXI_R2A_CTRL MT_WFDMA_EXT_CSR(0x500) +#define MT_WFDMA_AXI_R2A_CTRL_OUTSTAND_MASK GENMASK(4, 0) + #define MT_PCIE_RECOG_ID 0xd7090 #define MT_PCIE_RECOG_ID_MASK GENMASK(30, 0) #define MT_PCIE_RECOG_ID_SEM BIT(31) @@ -374,6 +453,9 @@ enum base_rev { #define MT_WFDMA0_PCIE1_BASE 0xd8000 #define MT_WFDMA0_PCIE1(ofs) (MT_WFDMA0_PCIE1_BASE + (ofs)) +#define MT_INT_PCIE1_SOURCE_CSR_EXT MT_WFDMA0_PCIE1(0x118) +#define MT_INT_PCIE1_MASK_CSR MT_WFDMA0_PCIE1(0x11c) + #define MT_WFDMA0_PCIE1_BUSY_ENA MT_WFDMA0_PCIE1(0x13c) #define MT_WFDMA0_PCIE1_BUSY_ENA_TX_FIFO0 BIT(0) #define MT_WFDMA0_PCIE1_BUSY_ENA_TX_FIFO1 BIT(1) @@ -394,6 +476,7 @@ enum base_rev { #define MT_MCUQ_RING_BASE(q) (MT_Q_BASE(q) + 0x300) #define MT_TXQ_RING_BASE(q) (MT_Q_BASE(__TXQ(q)) + 0x300) #define MT_RXQ_RING_BASE(q) (MT_Q_BASE(__RXQ(q)) + 0x500) +#define MT_RXQ_RRO_IND_RING_BASE MT_RRO_TOP(0x40) #define MT_MCUQ_EXT_CTRL(q) (MT_Q_BASE(q) + 0x600 + \ MT_MCUQ_ID(q) * 0x4) @@ -409,17 +492,27 @@ enum base_rev { #define MT_INT1_MASK_CSR MT_WFDMA0_PCIE1(0x204) #define MT_INT_RX_DONE_BAND0 BIT(12) -#define MT_INT_RX_DONE_BAND1 BIT(12) +#define MT_INT_RX_DONE_BAND1 BIT(13) /* for mt7992 */ #define MT_INT_RX_DONE_BAND2 BIT(13) #define MT_INT_RX_DONE_WM BIT(0) #define MT_INT_RX_DONE_WA BIT(1) #define MT_INT_RX_DONE_WA_MAIN BIT(2) -#define MT_INT_RX_DONE_WA_EXT BIT(2) +#define MT_INT_RX_DONE_WA_EXT BIT(3) /* for mt7992 */ #define MT_INT_RX_DONE_WA_TRI BIT(3) #define MT_INT_RX_TXFREE_MAIN BIT(17) #define MT_INT_RX_TXFREE_TRI BIT(15) +#define MT_INT_RX_DONE_BAND2_EXT BIT(23) +#define MT_INT_RX_TXFREE_EXT BIT(26) #define MT_INT_MCU_CMD BIT(29) +#define MT_INT_RX_DONE_RRO_BAND0 BIT(16) +#define MT_INT_RX_DONE_RRO_BAND1 BIT(16) +#define MT_INT_RX_DONE_RRO_BAND2 BIT(14) +#define MT_INT_RX_DONE_RRO_IND BIT(11) +#define MT_INT_RX_DONE_MSDU_PG_BAND0 BIT(18) +#define MT_INT_RX_DONE_MSDU_PG_BAND1 BIT(19) +#define MT_INT_RX_DONE_MSDU_PG_BAND2 BIT(23) + #define MT_INT_RX(q) (dev->q_int_mask[__RXQ(q)]) #define MT_INT_TX_MCU(q) (dev->q_int_mask[(q)]) @@ -427,20 +520,31 @@ enum base_rev { MT_INT_RX(MT_RXQ_MCU_WA)) #define MT_INT_BAND0_RX_DONE (MT_INT_RX(MT_RXQ_MAIN) | \ - MT_INT_RX(MT_RXQ_MAIN_WA)) + MT_INT_RX(MT_RXQ_MAIN_WA) | \ + MT_INT_RX(MT_RXQ_TXFREE_BAND0)) #define MT_INT_BAND1_RX_DONE (MT_INT_RX(MT_RXQ_BAND1) | \ MT_INT_RX(MT_RXQ_BAND1_WA) | \ - MT_INT_RX(MT_RXQ_MAIN_WA)) + MT_INT_RX(MT_RXQ_MAIN_WA) | \ + MT_INT_RX(MT_RXQ_TXFREE_BAND0)) #define MT_INT_BAND2_RX_DONE (MT_INT_RX(MT_RXQ_BAND2) | \ MT_INT_RX(MT_RXQ_BAND2_WA) | \ - MT_INT_RX(MT_RXQ_MAIN_WA)) + MT_INT_RX(MT_RXQ_MAIN_WA) | \ + MT_INT_RX(MT_RXQ_TXFREE_BAND0)) + +#define MT_INT_RRO_RX_DONE (MT_INT_RX(MT_RXQ_RRO_BAND0) | \ + MT_INT_RX(MT_RXQ_RRO_BAND1) | \ + MT_INT_RX(MT_RXQ_RRO_BAND2) | \ + MT_INT_RX(MT_RXQ_MSDU_PAGE_BAND0) | \ + MT_INT_RX(MT_RXQ_MSDU_PAGE_BAND1) | \ + MT_INT_RX(MT_RXQ_MSDU_PAGE_BAND2)) #define MT_INT_RX_DONE_ALL (MT_INT_RX_DONE_MCU | \ MT_INT_BAND0_RX_DONE | \ MT_INT_BAND1_RX_DONE | \ - MT_INT_BAND2_RX_DONE) + MT_INT_BAND2_RX_DONE | \ + MT_INT_RRO_RX_DONE) #define MT_INT_TX_DONE_FWDL BIT(26) #define MT_INT_TX_DONE_MCU_WM BIT(27) @@ -449,6 +553,10 @@ enum base_rev { #define MT_INT_TX_DONE_BAND1 BIT(31) #define MT_INT_TX_DONE_BAND2 BIT(15) +#define MT_INT_TX_RX_DONE_EXT (MT_INT_TX_DONE_BAND2 | \ + MT_INT_RX_DONE_BAND2_EXT | \ + MT_INT_RX_TXFREE_EXT) + #define MT_INT_TX_DONE_MCU (MT_INT_TX_MCU(MT_MCUQ_WA) | \ MT_INT_TX_MCU(MT_MCUQ_WM) | \ MT_INT_TX_MCU(MT_MCUQ_FWDL)) @@ -552,7 +660,12 @@ enum base_rev { #define MT_TOP_MISC MT_TOP(0xf0) #define MT_TOP_MISC_FW_STATE GENMASK(2, 0) +#define MT_PAD_GPIO 0x700056f0 +#define MT_PAD_GPIO_ADIE_COMB GENMASK(16, 15) + #define MT_HW_REV 0x70010204 +#define MT_HW_REV1 0x8a00 + #define MT_WF_SUBSYS_RST 0x70028600 /* PCIE MAC */ @@ -601,4 +714,11 @@ enum base_rev { #define MT_MCU_WM_EXCP_LR_CTRL MT_MCU_WM_EXCP(0x200) #define MT_MCU_WM_EXCP_LR_LOG MT_MCU_WM_EXCP(0x204) +/* CONN AFE CTL CON */ +#define MT_AFE_CTL_BASE 0x18043000 +#define MT_AFE_CTL_BAND(_band, ofs) (MT_AFE_CTL_BASE + \ + ((_band) * 0x1000) + (ofs)) +#define MT_AFE_CTL_BAND_PLL_03(_band) MT_AFE_CTL_BAND(_band, 0x2c) +#define MT_AFE_CTL_BAND_PLL_03_MSB_EN BIT(1) + #endif diff --git a/drivers/net/wireless/mediatek/mt76/sdio.c b/drivers/net/wireless/mediatek/mt76/sdio.c index 419723118ded8e..c52d550f0c32aa 100644 --- a/drivers/net/wireless/mediatek/mt76/sdio.c +++ b/drivers/net/wireless/mediatek/mt76/sdio.c @@ -481,21 +481,21 @@ static void mt76s_status_worker(struct mt76_worker *w) if (dev->drv->tx_status_data && ndata_frames > 0 && !test_and_set_bit(MT76_READING_STATS, &dev->phy.state) && !test_bit(MT76_STATE_SUSPEND, &dev->phy.state)) - ieee80211_queue_work(dev->hw, &dev->sdio.stat_work); + mt76_worker_schedule(&sdio->stat_worker); } while (nframes > 0); if (resched) mt76_worker_schedule(&dev->tx_worker); } -static void mt76s_tx_status_data(struct work_struct *work) +static void mt76s_tx_status_data(struct mt76_worker *worker) { struct mt76_sdio *sdio; struct mt76_dev *dev; u8 update = 1; u16 count = 0; - sdio = container_of(work, struct mt76_sdio, stat_work); + sdio = container_of(worker, struct mt76_sdio, stat_worker); dev = container_of(sdio, struct mt76_dev, sdio); while (true) { @@ -508,7 +508,7 @@ static void mt76s_tx_status_data(struct work_struct *work) } if (count && test_bit(MT76_STATE_RUNNING, &dev->phy.state)) - ieee80211_queue_work(dev->hw, &sdio->stat_work); + mt76_worker_schedule(&sdio->status_worker); else clear_bit(MT76_READING_STATS, &dev->phy.state); } @@ -600,8 +600,8 @@ void mt76s_deinit(struct mt76_dev *dev) mt76_worker_teardown(&sdio->txrx_worker); mt76_worker_teardown(&sdio->status_worker); mt76_worker_teardown(&sdio->net_worker); + mt76_worker_teardown(&sdio->stat_worker); - cancel_work_sync(&sdio->stat_work); clear_bit(MT76_READING_STATS, &dev->phy.state); mt76_tx_status_check(dev, true); @@ -644,10 +644,14 @@ int mt76s_init(struct mt76_dev *dev, struct sdio_func *func, if (err) return err; + err = mt76_worker_setup(dev->hw, &sdio->stat_worker, mt76s_tx_status_data, + "sdio-sta"); + if (err) + return err; + sched_set_fifo_low(sdio->status_worker.task); sched_set_fifo_low(sdio->net_worker.task); - - INIT_WORK(&sdio->stat_work, mt76s_tx_status_data); + sched_set_fifo_low(sdio->stat_worker.task); dev->queue_ops = &sdio_queue_ops; dev->bus = bus_ops; diff --git a/drivers/net/wireless/ralink/rt2x00/rt2800lib.c b/drivers/net/wireless/ralink/rt2x00/rt2800lib.c index 84b218adbaaf19..aaf31857ae1efb 100644 --- a/drivers/net/wireless/ralink/rt2x00/rt2800lib.c +++ b/drivers/net/wireless/ralink/rt2x00/rt2800lib.c @@ -1328,8 +1328,8 @@ static bool rt2800_watchdog_dma_busy(struct rt2x00_dev *rt2x00dev) else rt2x00dev->txdma_busy = 0; - busy_rx = rt2x00dev->rxdma_busy > 30 ? true : false; - busy_tx = rt2x00dev->txdma_busy > 30 ? true : false; + busy_rx = rt2x00dev->rxdma_busy > 30; + busy_tx = rt2x00dev->txdma_busy > 30; if (!busy_rx && !busy_tx) return false; @@ -1353,10 +1353,10 @@ void rt2800_watchdog(struct rt2x00_dev *rt2x00dev) if (test_bit(DEVICE_STATE_SCANNING, &rt2x00dev->flags)) return; - if (modparam_watchdog & RT2800_WATCHDOG_DMA_BUSY) + if (rt2x00dev->link.watchdog & RT2800_WATCHDOG_DMA_BUSY) reset = rt2800_watchdog_dma_busy(rt2x00dev); - if (modparam_watchdog & RT2800_WATCHDOG_HANG) + if (rt2x00dev->link.watchdog & RT2800_WATCHDOG_HANG) reset = rt2800_watchdog_hung(rt2x00dev) || reset; if (reset) @@ -12065,14 +12065,13 @@ int rt2800_probe_hw(struct rt2x00_dev *rt2x00dev) __set_bit(REQUIRE_TASKLET_CONTEXT, &rt2x00dev->cap_flags); } + rt2x00dev->link.watchdog = modparam_watchdog; /* USB NICs don't support DMA watchdog as INT_SOURCE_CSR is invalid */ if (rt2x00_is_usb(rt2x00dev)) - modparam_watchdog &= ~RT2800_WATCHDOG_DMA_BUSY; - if (modparam_watchdog) { + rt2x00dev->link.watchdog &= ~RT2800_WATCHDOG_DMA_BUSY; + if (rt2x00dev->link.watchdog) { __set_bit(CAPABILITY_RESTART_HW, &rt2x00dev->cap_flags); rt2x00dev->link.watchdog_interval = msecs_to_jiffies(100); - } else { - rt2x00dev->link.watchdog_disabled = true; } /* diff --git a/drivers/net/wireless/ralink/rt2x00/rt2x00.h b/drivers/net/wireless/ralink/rt2x00/rt2x00.h index 62fed38f41c080..82af01448a0a8d 100644 --- a/drivers/net/wireless/ralink/rt2x00/rt2x00.h +++ b/drivers/net/wireless/ralink/rt2x00/rt2x00.h @@ -334,7 +334,7 @@ struct link { */ struct delayed_work watchdog_work; unsigned int watchdog_interval; - bool watchdog_disabled; + unsigned int watchdog; /* * Work structure for scheduling periodic AGC adjustments. diff --git a/drivers/net/wireless/ralink/rt2x00/rt2x00link.c b/drivers/net/wireless/ralink/rt2x00/rt2x00link.c index 6cf7e7c997c2bc..fb23d409fba87d 100644 --- a/drivers/net/wireless/ralink/rt2x00/rt2x00link.c +++ b/drivers/net/wireless/ralink/rt2x00/rt2x00link.c @@ -384,7 +384,7 @@ void rt2x00link_start_watchdog(struct rt2x00_dev *rt2x00dev) struct link *link = &rt2x00dev->link; if (test_bit(DEVICE_STATE_PRESENT, &rt2x00dev->flags) && - rt2x00dev->ops->lib->watchdog && !link->watchdog_disabled) + rt2x00dev->ops->lib->watchdog && link->watchdog) ieee80211_queue_delayed_work(rt2x00dev->hw, &link->watchdog_work, link->watchdog_interval); diff --git a/drivers/net/wireless/ralink/rt2x00/rt2x00queue.c b/drivers/net/wireless/ralink/rt2x00/rt2x00queue.c index 98df0aef816850..013003777fee19 100644 --- a/drivers/net/wireless/ralink/rt2x00/rt2x00queue.c +++ b/drivers/net/wireless/ralink/rt2x00/rt2x00queue.c @@ -416,9 +416,6 @@ static void rt2x00queue_create_tx_descriptor(struct rt2x00_dev *rt2x00dev, __set_bit(ENTRY_TXD_RTS_FRAME, &txdesc->flags); else __set_bit(ENTRY_TXD_CTS_FRAME, &txdesc->flags); - if (tx_info->control.rts_cts_rate_idx >= 0) - rate = - ieee80211_get_rts_cts_rate(rt2x00dev->hw, tx_info); } /* diff --git a/drivers/net/wireless/realtek/rtlwifi/pci.c b/drivers/net/wireless/realtek/rtlwifi/pci.c index b163a069660b83..96ce05bcf0b3a4 100644 --- a/drivers/net/wireless/realtek/rtlwifi/pci.c +++ b/drivers/net/wireless/realtek/rtlwifi/pci.c @@ -64,7 +64,7 @@ static void _rtl_pci_update_default_setting(struct ieee80211_hw *hw) struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw)); struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw)); u8 pcibridge_vendor = pcipriv->ndis_adapter.pcibridge_vendor; - u8 init_aspm; + u16 init_aspm; ppsc->reg_rfps_level = 0; ppsc->support_aspm = false; @@ -150,9 +150,10 @@ static void _rtl_pci_update_default_setting(struct ieee80211_hw *hw) /* toshiba aspm issue, toshiba will set aspm selfly * so we should not set aspm in driver */ - pci_read_config_byte(rtlpci->pdev, 0x80, &init_aspm); + pcie_capability_read_word(rtlpci->pdev, PCI_EXP_LNKCTL, &init_aspm); if (rtlpriv->rtlhal.hw_type == HARDWARE_TYPE_RTL8192SE && - init_aspm == 0x43) + ((u8)init_aspm) == (PCI_EXP_LNKCTL_ASPM_L0S | + PCI_EXP_LNKCTL_ASPM_L1 | PCI_EXP_LNKCTL_CCC)) ppsc->support_aspm = false; } @@ -163,21 +164,29 @@ static bool _rtl_pci_platform_switch_device_pci_aspm( struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw)); struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw)); + value &= PCI_EXP_LNKCTL_ASPMC; + if (rtlhal->hw_type != HARDWARE_TYPE_RTL8192SE) - value |= 0x40; + value |= PCI_EXP_LNKCTL_CCC; - pci_write_config_byte(rtlpci->pdev, 0x80, value); + pcie_capability_clear_and_set_word(rtlpci->pdev, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_ASPMC | value, + value); return false; } -/*When we set 0x01 to enable clk request. Set 0x0 to disable clk req.*/ -static void _rtl_pci_switch_clk_req(struct ieee80211_hw *hw, u8 value) +/* @value is PCI_EXP_LNKCTL_CLKREQ_EN or 0 to enable/disable clk request. */ +static void _rtl_pci_switch_clk_req(struct ieee80211_hw *hw, u16 value) { struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw)); struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw)); - pci_write_config_byte(rtlpci->pdev, 0x81, value); + value &= PCI_EXP_LNKCTL_CLKREQ_EN; + + pcie_capability_clear_and_set_word(rtlpci->pdev, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_CLKREQ_EN, + value); if (rtlhal->hw_type == HARDWARE_TYPE_RTL8192SE) udelay(100); @@ -191,13 +200,10 @@ static void rtl_pci_disable_aspm(struct ieee80211_hw *hw) struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw)); struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw)); u8 pcibridge_vendor = pcipriv->ndis_adapter.pcibridge_vendor; - u8 num4bytes = pcipriv->ndis_adapter.num4bytes; /*Retrieve original configuration settings. */ u8 linkctrl_reg = pcipriv->ndis_adapter.linkctrl_reg; - u16 pcibridge_linkctrlreg = pcipriv->ndis_adapter. - pcibridge_linkctrlreg; u16 aspmlevel = 0; - u8 tmp_u1b = 0; + u16 tmp_u1b = 0; if (!ppsc->support_aspm) return; @@ -215,21 +221,13 @@ static void rtl_pci_disable_aspm(struct ieee80211_hw *hw) } /*for promising device will in L0 state after an I/O. */ - pci_read_config_byte(rtlpci->pdev, 0x80, &tmp_u1b); + pcie_capability_read_word(rtlpci->pdev, PCI_EXP_LNKCTL, &tmp_u1b); /*Set corresponding value. */ - aspmlevel |= BIT(0) | BIT(1); + aspmlevel |= PCI_EXP_LNKCTL_ASPM_L0S | PCI_EXP_LNKCTL_ASPM_L1; linkctrl_reg &= ~aspmlevel; - pcibridge_linkctrlreg &= ~(BIT(0) | BIT(1)); _rtl_pci_platform_switch_device_pci_aspm(hw, linkctrl_reg); - udelay(50); - - /*4 Disable Pci Bridge ASPM */ - pci_write_config_byte(rtlpci->pdev, (num4bytes << 2), - pcibridge_linkctrlreg); - - udelay(50); } /*Enable RTL8192SE ASPM & Enable Pci Bridge ASPM for @@ -244,9 +242,7 @@ static void rtl_pci_enable_aspm(struct ieee80211_hw *hw) struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw)); struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw)); u8 pcibridge_vendor = pcipriv->ndis_adapter.pcibridge_vendor; - u8 num4bytes = pcipriv->ndis_adapter.num4bytes; u16 aspmlevel; - u8 u_pcibridge_aspmsetting; u8 u_device_aspmsetting; if (!ppsc->support_aspm) @@ -258,25 +254,6 @@ static void rtl_pci_enable_aspm(struct ieee80211_hw *hw) return; } - /*4 Enable Pci Bridge ASPM */ - - u_pcibridge_aspmsetting = - pcipriv->ndis_adapter.pcibridge_linkctrlreg | - rtlpci->const_hostpci_aspm_setting; - - if (pcibridge_vendor == PCI_BRIDGE_VENDOR_INTEL) - u_pcibridge_aspmsetting &= ~BIT(0); - - pci_write_config_byte(rtlpci->pdev, (num4bytes << 2), - u_pcibridge_aspmsetting); - - rtl_dbg(rtlpriv, COMP_INIT, DBG_LOUD, - "PlatformEnableASPM(): Write reg[%x] = %x\n", - (pcipriv->ndis_adapter.pcibridge_pciehdr_offset + 0x10), - u_pcibridge_aspmsetting); - - udelay(50); - /*Get ASPM level (with/without Clock Req) */ aspmlevel = rtlpci->const_devicepci_aspm_setting; u_device_aspmsetting = pcipriv->ndis_adapter.linkctrl_reg; @@ -290,7 +267,8 @@ static void rtl_pci_enable_aspm(struct ieee80211_hw *hw) if (ppsc->reg_rfps_level & RT_RF_OFF_LEVL_CLK_REQ) { _rtl_pci_switch_clk_req(hw, (ppsc->reg_rfps_level & - RT_RF_OFF_LEVL_CLK_REQ) ? 1 : 0); + RT_RF_OFF_LEVL_CLK_REQ) ? + PCI_EXP_LNKCTL_CLKREQ_EN : 0); RT_SET_PS_LEVEL(ppsc, RT_RF_OFF_LEVL_CLK_REQ); } udelay(100); @@ -357,22 +335,6 @@ static bool rtl_pci_check_buddy_priv(struct ieee80211_hw *hw, return tpriv != NULL; } -static void rtl_pci_get_linkcontrol_field(struct ieee80211_hw *hw) -{ - struct rtl_pci_priv *pcipriv = rtl_pcipriv(hw); - struct rtl_pci *rtlpci = rtl_pcidev(pcipriv); - u8 capabilityoffset = pcipriv->ndis_adapter.pcibridge_pciehdr_offset; - u8 linkctrl_reg; - u8 num4bbytes; - - num4bbytes = (capabilityoffset + 0x10) / 4; - - /*Read Link Control Register */ - pci_read_config_byte(rtlpci->pdev, (num4bbytes << 2), &linkctrl_reg); - - pcipriv->ndis_adapter.pcibridge_linkctrlreg = linkctrl_reg; -} - static void rtl_pci_parse_configuration(struct pci_dev *pdev, struct ieee80211_hw *hw) { @@ -389,9 +351,8 @@ static void rtl_pci_parse_configuration(struct pci_dev *pdev, rtl_dbg(rtlpriv, COMP_INIT, DBG_TRACE, "Link Control Register =%x\n", pcipriv->ndis_adapter.linkctrl_reg); - pci_read_config_byte(pdev, 0x98, &tmp); - tmp |= BIT(4); - pci_write_config_byte(pdev, 0x98, tmp); + pcie_capability_set_word(pdev, PCI_EXP_DEVCTL2, + PCI_EXP_DEVCTL2_COMP_TMOUT_DIS); tmp = 0x17; pci_write_config_byte(pdev, 0x70f, tmp); @@ -2007,7 +1968,6 @@ static bool _rtl_pci_find_adapter(struct pci_dev *pdev, */ if (bridge_pdev) { /*find bridge info if available */ - pcipriv->ndis_adapter.pcibridge_vendorid = bridge_pdev->vendor; for (tmp = 0; tmp < PCI_BRIDGE_VENDOR_MAX; tmp++) { if (bridge_pdev->vendor == pcibridge_vendors[tmp]) { pcipriv->ndis_adapter.pcibridge_vendor = tmp; @@ -2027,12 +1987,6 @@ static bool _rtl_pci_find_adapter(struct pci_dev *pdev, PCI_SLOT(bridge_pdev->devfn); pcipriv->ndis_adapter.pcibridge_funcnum = PCI_FUNC(bridge_pdev->devfn); - pcipriv->ndis_adapter.pcibridge_pciehdr_offset = - pci_pcie_cap(bridge_pdev); - pcipriv->ndis_adapter.num4bytes = - (pcipriv->ndis_adapter.pcibridge_pciehdr_offset + 0x10) / 4; - - rtl_pci_get_linkcontrol_field(hw); if (pcipriv->ndis_adapter.pcibridge_vendor == PCI_BRIDGE_VENDOR_AMD) { @@ -2049,13 +2003,11 @@ static bool _rtl_pci_find_adapter(struct pci_dev *pdev, pdev->vendor, pcipriv->ndis_adapter.linkctrl_reg); rtl_dbg(rtlpriv, COMP_INIT, DBG_DMESG, - "pci_bridge busnumber:devnumber:funcnumber:vendor:pcie_cap:link_ctl_reg:amd %d:%d:%d:%x:%x:%x:%x\n", + "pci_bridge busnumber:devnumber:funcnumber:vendor:amd %d:%d:%d:%x:%x\n", pcipriv->ndis_adapter.pcibridge_busnum, pcipriv->ndis_adapter.pcibridge_devnum, pcipriv->ndis_adapter.pcibridge_funcnum, pcibridge_vendors[pcipriv->ndis_adapter.pcibridge_vendor], - pcipriv->ndis_adapter.pcibridge_pciehdr_offset, - pcipriv->ndis_adapter.pcibridge_linkctrlreg, pcipriv->ndis_adapter.amd_l1_patch); rtl_pci_parse_configuration(pdev, hw); diff --git a/drivers/net/wireless/realtek/rtlwifi/pci.h b/drivers/net/wireless/realtek/rtlwifi/pci.h index 4725d43609fd24..e8fa022df8b4c0 100644 --- a/drivers/net/wireless/realtek/rtlwifi/pci.h +++ b/drivers/net/wireless/realtek/rtlwifi/pci.h @@ -44,18 +44,6 @@ #define ATI_DEVICE_ID 0x7914 #define AMD_VENDOR_ID 0x1022 -#define PCI_MAX_BRIDGE_NUMBER 255 -#define PCI_MAX_DEVICES 32 -#define PCI_MAX_FUNCTION 8 - -#define PCI_CONF_ADDRESS 0x0CF8 /*PCI Configuration Space Address */ -#define PCI_CONF_DATA 0x0CFC /*PCI Configuration Space Data */ - -#define PCI_CLASS_BRIDGE_DEV 0x06 -#define PCI_SUBCLASS_BR_PCI_TO_PCI 0x04 -#define PCI_CAPABILITY_ID_PCI_EXPRESS 0x10 -#define PCI_CAP_ID_EXP 0x10 - #define U1DONTCARE 0xFF #define U2DONTCARE 0xFFFF #define U4DONTCARE 0xFFFFFFFF @@ -113,11 +101,6 @@ enum pci_bridge_vendor { PCI_BRIDGE_VENDOR_MAX, }; -struct rtl_pci_capabilities_header { - u8 capability_id; - u8 next; -}; - /* In new TRX flow, Buffer_desc is new concept * But TX wifi info == TX descriptor in old flow * RX wifi info == RX descriptor in old flow @@ -232,13 +215,6 @@ struct mp_adapter { u8 pcibridge_funcnum; u8 pcibridge_vendor; - u16 pcibridge_vendorid; - u16 pcibridge_deviceid; - - u8 num4bytes; - - u8 pcibridge_pciehdr_offset; - u8 pcibridge_linkctrlreg; bool amd_l1_patch; }; diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/hw.c index 1633328bc3d1dd..f4b232f038a936 100644 --- a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/hw.c +++ b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/hw.c @@ -2270,67 +2270,27 @@ static void _rtl8821ae_clear_pci_pme_status(struct ieee80211_hw *hw) { struct rtl_priv *rtlpriv = rtl_priv(hw); struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw)); - u16 cap_hdr; - u8 cap_pointer; - u8 cap_id = 0xff; - u8 pmcs_reg; - u8 cnt = 0; + struct pci_dev *pdev = rtlpci->pdev; + u16 pmcs_reg; + u8 pm_cap; - /* Get the Capability pointer first, - * the Capability Pointer is located at - * offset 0x34 from the Function Header */ - - pci_read_config_byte(rtlpci->pdev, 0x34, &cap_pointer); - rtl_dbg(rtlpriv, COMP_INIT, DBG_LOUD, - "PCI configuration 0x34 = 0x%2x\n", cap_pointer); - - do { - pci_read_config_word(rtlpci->pdev, cap_pointer, &cap_hdr); - cap_id = cap_hdr & 0xFF; - - rtl_dbg(rtlpriv, COMP_INIT, DBG_LOUD, - "in pci configuration, cap_pointer%x = %x\n", - cap_pointer, cap_id); - - if (cap_id == 0x01) { - break; - } else { - /* point to next Capability */ - cap_pointer = (cap_hdr >> 8) & 0xFF; - /* 0: end of pci capability, 0xff: invalid value */ - if (cap_pointer == 0x00 || cap_pointer == 0xff) { - cap_id = 0xff; - break; - } - } - } while (cnt++ < 200); - - if (cap_id == 0x01) { - /* Get the PM CSR (Control/Status Register), - * The PME_Status is located at PM Capatibility offset 5, bit 7 - */ - pci_read_config_byte(rtlpci->pdev, cap_pointer + 5, &pmcs_reg); - - if (pmcs_reg & BIT(7)) { - /* PME event occured, clear the PM_Status by write 1 */ - pmcs_reg = pmcs_reg | BIT(7); - - pci_write_config_byte(rtlpci->pdev, cap_pointer + 5, - pmcs_reg); - /* Read it back to check */ - pci_read_config_byte(rtlpci->pdev, cap_pointer + 5, - &pmcs_reg); - rtl_dbg(rtlpriv, COMP_INIT, DBG_DMESG, - "Clear PME status 0x%2x to 0x%2x\n", - cap_pointer + 5, pmcs_reg); - } else { - rtl_dbg(rtlpriv, COMP_INIT, DBG_DMESG, - "PME status(0x%2x) = 0x%2x\n", - cap_pointer + 5, pmcs_reg); - } - } else { + pm_cap = pci_find_capability(pdev, PCI_CAP_ID_PM); + if (!pm_cap) { rtl_dbg(rtlpriv, COMP_INIT, DBG_WARNING, "Cannot find PME Capability\n"); + return; + } + + pci_read_config_word(pdev, pm_cap + PCI_PM_CTRL, &pmcs_reg); + if (pmcs_reg & PCI_PM_CTRL_PME_STATUS) { + /* Clear PME_Status with write */ + pci_write_config_word(pdev, pm_cap + PCI_PM_CTRL, pmcs_reg); + pci_read_config_word(pdev, pm_cap + PCI_PM_CTRL, &pmcs_reg); + rtl_dbg(rtlpriv, COMP_INIT, DBG_DMESG, + "Cleared PME status, PMCS reg = 0x%4x\n", pmcs_reg); + } else { + rtl_dbg(rtlpriv, COMP_INIT, DBG_DMESG, + "PMCS reg = 0x%4x\n", pmcs_reg); } } diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/phy.c b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/phy.c index 5323ead30db038..68c3fb0395ce4a 100644 --- a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/phy.c +++ b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/phy.c @@ -29,9 +29,10 @@ static void _rtl8821ae_phy_rf_serial_write(struct ieee80211_hw *hw, u32 data); static u32 _rtl8821ae_phy_calculate_bit_shift(u32 bitmask) { - u32 i = ffs(bitmask); + if (WARN_ON_ONCE(!bitmask)) + return 0; - return i ? i - 1 : 32; + return __ffs(bitmask); } static bool _rtl8821ae_phy_bb8821a_config_parafile(struct ieee80211_hw *hw); /*static bool _rtl8812ae_phy_config_mac_with_headerfile(struct ieee80211_hw *hw);*/ @@ -2038,15 +2039,9 @@ static bool _rtl8821ae_phy_config_bb_with_pgheaderfile(struct ieee80211_hw *hw, /*don't need the hw_body*/ if (!_rtl8821ae_check_condition(hw, v1)) { i += 2; /* skip the pair of expression*/ - v1 = array[i]; v2 = array[i+1]; - v3 = array[i+2]; - while (v2 != 0xDEAD) { + while (v2 != 0xDEAD) i += 3; - v1 = array[i]; - v2 = array[i+1]; - v3 = array[i+2]; - } } } } @@ -3543,7 +3538,6 @@ u8 rtl8821ae_phy_sw_chnl(struct ieee80211_hw *hw) struct rtl_phy *rtlphy = &rtlpriv->phy; struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw)); u32 timeout = 1000, timecount = 0; - u8 channel = rtlphy->current_channel; if (rtlphy->sw_chnl_inprogress) return 0; @@ -3566,8 +3560,6 @@ u8 rtl8821ae_phy_sw_chnl(struct ieee80211_hw *hw) rtl8821ae_phy_switch_wirelessband(hw, BAND_ON_2_4G); rtlphy->sw_chnl_inprogress = true; - if (channel == 0) - channel = 1; rtl_dbg(rtlpriv, COMP_SCAN, DBG_TRACE, "switch to channel%d, band type is %d\n", diff --git a/drivers/net/wireless/realtek/rtw88/debug.c b/drivers/net/wireless/realtek/rtw88/debug.c index 35bc37a3c469db..1b2ad81838be38 100644 --- a/drivers/net/wireless/realtek/rtw88/debug.c +++ b/drivers/net/wireless/realtek/rtw88/debug.c @@ -1314,8 +1314,8 @@ void rtw_debugfs_init(struct rtw_dev *rtwdev) #ifdef CONFIG_RTW88_DEBUG -void __rtw_dbg(struct rtw_dev *rtwdev, enum rtw_debug_mask mask, - const char *fmt, ...) +void rtw_dbg(struct rtw_dev *rtwdev, enum rtw_debug_mask mask, + const char *fmt, ...) { struct va_format vaf = { .fmt = fmt, @@ -1330,6 +1330,6 @@ void __rtw_dbg(struct rtw_dev *rtwdev, enum rtw_debug_mask mask, va_end(args); } -EXPORT_SYMBOL(__rtw_dbg); +EXPORT_SYMBOL(rtw_dbg); #endif /* CONFIG_RTW88_DEBUG */ diff --git a/drivers/net/wireless/realtek/rtw88/debug.h b/drivers/net/wireless/realtek/rtw88/debug.h index a03ced11bbe042..f20c0471c82a4a 100644 --- a/drivers/net/wireless/realtek/rtw88/debug.h +++ b/drivers/net/wireless/realtek/rtw88/debug.h @@ -43,10 +43,8 @@ static inline void rtw_debugfs_init(struct rtw_dev *rtwdev) {} #ifdef CONFIG_RTW88_DEBUG __printf(3, 4) -void __rtw_dbg(struct rtw_dev *rtwdev, enum rtw_debug_mask mask, - const char *fmt, ...); - -#define rtw_dbg(rtwdev, a...) __rtw_dbg(rtwdev, ##a) +void rtw_dbg(struct rtw_dev *rtwdev, enum rtw_debug_mask mask, + const char *fmt, ...); static inline bool rtw_dbg_is_enabled(struct rtw_dev *rtwdev, enum rtw_debug_mask mask) diff --git a/drivers/net/wireless/realtek/rtw88/main.c b/drivers/net/wireless/realtek/rtw88/main.c index 4a33d2e47f33ff..6d22628129d0d5 100644 --- a/drivers/net/wireless/realtek/rtw88/main.c +++ b/drivers/net/wireless/realtek/rtw88/main.c @@ -2008,6 +2008,11 @@ static int rtw_chip_efuse_info_setup(struct rtw_dev *rtwdev) efuse->ext_pa_5g = efuse->pa_type_5g & BIT(0) ? 1 : 0; efuse->ext_lna_2g = efuse->lna_type_5g & BIT(3) ? 1 : 0; + if (!is_valid_ether_addr(efuse->addr)) { + eth_random_addr(efuse->addr); + dev_warn(rtwdev->dev, "efuse MAC invalid, using random\n"); + } + out_disable: rtw_chip_efuse_disable(rtwdev); diff --git a/drivers/net/wireless/realtek/rtw88/sdio.c b/drivers/net/wireless/realtek/rtw88/sdio.c index 2c1fb2dabd40a4..0cae5746f540fa 100644 --- a/drivers/net/wireless/realtek/rtw88/sdio.c +++ b/drivers/net/wireless/realtek/rtw88/sdio.c @@ -500,19 +500,40 @@ static u32 rtw_sdio_get_tx_addr(struct rtw_dev *rtwdev, size_t size, static int rtw_sdio_read_port(struct rtw_dev *rtwdev, u8 *buf, size_t count) { struct rtw_sdio *rtwsdio = (struct rtw_sdio *)rtwdev->priv; + struct mmc_host *host = rtwsdio->sdio_func->card->host; bool bus_claim = rtw_sdio_bus_claim_needed(rtwsdio); u32 rxaddr = rtwsdio->rx_addr++; - int ret; + int ret = 0, err; + size_t bytes; if (bus_claim) sdio_claim_host(rtwsdio->sdio_func); - ret = sdio_memcpy_fromio(rtwsdio->sdio_func, buf, - RTW_SDIO_ADDR_RX_RX0FF_GEN(rxaddr), count); - if (ret) - rtw_warn(rtwdev, - "Failed to read %zu byte(s) from SDIO port 0x%08x", - count, rxaddr); + while (count > 0) { + bytes = min_t(size_t, host->max_req_size, count); + + err = sdio_memcpy_fromio(rtwsdio->sdio_func, buf, + RTW_SDIO_ADDR_RX_RX0FF_GEN(rxaddr), + bytes); + if (err) { + rtw_warn(rtwdev, + "Failed to read %zu byte(s) from SDIO port 0x%08x: %d", + bytes, rxaddr, err); + + /* Signal to the caller that reading did not work and + * that the data in the buffer is short/corrupted. + */ + ret = err; + + /* Don't stop here - instead drain the remaining data + * from the card's buffer, else the card will return + * corrupt data for the next rtw_sdio_read_port() call. + */ + } + + count -= bytes; + buf += bytes; + } if (bus_claim) sdio_release_host(rtwsdio->sdio_func); diff --git a/drivers/net/wireless/realtek/rtw89/coex.c b/drivers/net/wireless/realtek/rtw89/coex.c index b842cd9a86f838..9c0db35d3e13c6 100644 --- a/drivers/net/wireless/realtek/rtw89/coex.c +++ b/drivers/net/wireless/realtek/rtw89/coex.c @@ -5352,6 +5352,10 @@ void rtw89_btc_ntfy_scan_start(struct rtw89_dev *rtwdev, u8 phy_idx, u8 band) rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], %s(): phy_idx=%d, band=%d\n", __func__, phy_idx, band); + + if (phy_idx >= RTW89_PHY_MAX) + return; + btc->dm.cnt_notify[BTC_NCNT_SCAN_START]++; wl->status.map.scan = true; wl->scan_info.band[phy_idx] = band; @@ -5396,6 +5400,10 @@ void rtw89_btc_ntfy_switch_band(struct rtw89_dev *rtwdev, u8 phy_idx, u8 band) rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], %s(): phy_idx=%d, band=%d\n", __func__, phy_idx, band); + + if (phy_idx >= RTW89_PHY_MAX) + return; + btc->dm.cnt_notify[BTC_NCNT_SWITCH_BAND]++; wl->scan_info.band[phy_idx] = band; diff --git a/drivers/net/wireless/realtek/rtw89/core.c b/drivers/net/wireless/realtek/rtw89/core.c index 74bf2964382396..fd527a24999641 100644 --- a/drivers/net/wireless/realtek/rtw89/core.c +++ b/drivers/net/wireless/realtek/rtw89/core.c @@ -2883,9 +2883,6 @@ void rtw89_roc_start(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif) lockdep_assert_held(&rtwdev->mutex); - ieee80211_queue_delayed_work(hw, &rtwvif->roc.roc_work, - msecs_to_jiffies(rtwvif->roc.duration)); - rtw89_leave_ips_by_hwflags(rtwdev); rtw89_leave_lps(rtwdev); rtw89_chanctx_pause(rtwdev, RTW89_CHANCTX_PAUSE_REASON_ROC); @@ -2907,6 +2904,9 @@ void rtw89_roc_start(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif) B_AX_A_UC_CAM_MATCH | B_AX_A_BC_CAM_MATCH); ieee80211_ready_on_channel(hw); + cancel_delayed_work(&rtwvif->roc.roc_work); + ieee80211_queue_delayed_work(hw, &rtwvif->roc.roc_work, + msecs_to_jiffies(rtwvif->roc.duration)); } void rtw89_roc_end(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif) @@ -3129,6 +3129,7 @@ static void rtw89_track_work(struct work_struct *work) rtw89_phy_tx_path_div_track(rtwdev); rtw89_phy_antdiv_track(rtwdev); rtw89_phy_ul_tb_ctrl_track(rtwdev); + rtw89_phy_edcca_track(rtwdev); rtw89_tas_track(rtwdev); rtw89_chanctx_track(rtwdev); @@ -3955,10 +3956,7 @@ int rtw89_core_start(struct rtw89_dev *rtwdev) /* efuse process */ /* pre-config BB/RF, BB reset/RFC reset */ - ret = rtw89_chip_disable_bb_rf(rtwdev); - if (ret) - return ret; - ret = rtw89_chip_enable_bb_rf(rtwdev); + ret = rtw89_chip_reset_bb_rf(rtwdev); if (ret) return ret; diff --git a/drivers/net/wireless/realtek/rtw89/core.h b/drivers/net/wireless/realtek/rtw89/core.h index 6948ffe0f20610..21421980fd0674 100644 --- a/drivers/net/wireless/realtek/rtw89/core.h +++ b/drivers/net/wireless/realtek/rtw89/core.h @@ -17,6 +17,8 @@ struct rtw89_pci_info; struct rtw89_mac_gen_def; struct rtw89_phy_gen_def; struct rtw89_efuse_block_cfg; +struct rtw89_fw_txpwr_track_cfg; +struct rtw89_phy_rfk_log_fmt; extern const struct ieee80211_ops rtw89_ops; @@ -38,6 +40,8 @@ extern const struct ieee80211_ops rtw89_ops; #define RSSI_FACTOR 1 #define RTW89_RSSI_RAW_TO_DBM(rssi) ((s8)((rssi) >> RSSI_FACTOR) - MAX_RSSI) #define RTW89_TX_DIV_RSSI_RAW_TH (2 << RSSI_FACTOR) +#define DELTA_SWINGIDX_SIZE 30 + #define RTW89_RADIOTAP_ROOM_HE sizeof(struct ieee80211_radiotap_he) #define RTW89_RADIOTAP_ROOM_EHT \ (sizeof(struct ieee80211_radiotap_tlv) + \ @@ -3292,6 +3296,8 @@ struct rtw89_dle_size { u16 pge_size; u16 lnk_pge_num; u16 unlnk_pge_num; + /* for WiFi 7 chips below */ + u32 srt_ofst; }; struct rtw89_wde_quota { @@ -3314,6 +3320,26 @@ struct rtw89_ple_quota { u16 wd_rel; u16 cpu_io; u16 tx_rpt; + /* for WiFi 7 chips below */ + u16 h2d; +}; + +struct rtw89_rsvd_quota { + u16 mpdu_info_tbl; + u16 b0_csi; + u16 b1_csi; + u16 b0_lmr; + u16 b1_lmr; + u16 b0_ftm; + u16 b1_ftm; + u16 b0_smr; + u16 b1_smr; + u16 others; +}; + +struct rtw89_dle_rsvd_size { + u32 srt_ofst; + u32 size; }; struct rtw89_dle_mem { @@ -3324,6 +3350,10 @@ struct rtw89_dle_mem { const struct rtw89_wde_quota *wde_max_qt; const struct rtw89_ple_quota *ple_min_qt; const struct rtw89_ple_quota *ple_max_qt; + /* for WiFi 7 chips below */ + const struct rtw89_rsvd_quota *rsvd_qt; + const struct rtw89_dle_rsvd_size *rsvd0_size; + const struct rtw89_dle_rsvd_size *rsvd1_size; }; struct rtw89_reg_def { @@ -3350,6 +3380,12 @@ struct rtw89_reg5_def { u32 data; }; +struct rtw89_reg_imr { + u32 addr; + u32 clr; + u32 set; +}; + struct rtw89_phy_table { const struct rtw89_reg2_def *regs; u32 n_regs; @@ -3559,6 +3595,11 @@ struct rtw89_imr_info { u32 tmac_imr_set; }; +struct rtw89_imr_table { + const struct rtw89_reg_imr *regs; + u32 n_regs; +}; + struct rtw89_xtal_info { u32 xcap_reg; u32 sc_xo_mask; @@ -3590,6 +3631,22 @@ struct rtw89_dig_regs { struct rtw89_reg_def p1_s20_pagcugc_en; }; +struct rtw89_edcca_regs { + u32 edcca_level; + u32 edcca_mask; + u32 edcca_p_mask; + u32 ppdu_level; + u32 ppdu_mask; + u32 rpt_a; + u32 rpt_b; + u32 rpt_sel; + u32 rpt_sel_mask; + u32 rpt_sel_be; + u32 rpt_sel_be_mask; + u32 tx_collision_t2r_st; + u32 tx_collision_t2r_st_mask; +}; + struct rtw89_phy_ul_tb_info { bool dyn_tb_tri_en; u8 def_if_bandedge; @@ -3650,8 +3707,8 @@ struct rtw89_chip_info { u32 rsvd_ple_ofst; const struct rtw89_hfc_param_ini *hfc_param_ini; const struct rtw89_dle_mem *dle_mem; - u8 wde_qempty_acq_num; - u8 wde_qempty_mgq_sel; + u8 wde_qempty_acq_grpnum; + u8 wde_qempty_mgq_grpsel; u32 rf_base_addr[2]; u8 support_chanctx_num; u8 support_bands; @@ -3737,11 +3794,13 @@ struct rtw89_chip_info { const struct rtw89_reg_def *dcfo_comp; u8 dcfo_comp_sft; const struct rtw89_imr_info *imr_info; + const struct rtw89_imr_table *imr_dmac_table; + const struct rtw89_imr_table *imr_cmac_table; const struct rtw89_rrsr_cfgs *rrsr_cfgs; struct rtw89_reg_def bss_clr_vld; u32 bss_clr_map_reg; u32 dma_ch_mask; - u32 edcca_lvl_reg; + const struct rtw89_edcca_regs *edcca_regs; const struct wiphy_wowlan_support *wowlan_stub; const struct rtw89_xtal_info *xtal_info; }; @@ -3765,8 +3824,10 @@ enum rtw89_hcifc_mode { }; struct rtw89_dle_info { + const struct rtw89_rsvd_quota *rsvd_qt; enum rtw89_qta_mode qta_mode; u16 ple_pg_size; + u16 ple_free_pg; u16 c0_rx_qta; u16 c1_rx_qta; }; @@ -3891,6 +3952,8 @@ struct rtw89_fw_elm_info { struct rtw89_phy_table *bb_gain; struct rtw89_phy_table *rf_radio[RF_PATH_MAX]; struct rtw89_phy_table *rf_nctl; + struct rtw89_fw_txpwr_track_cfg *txpwr_trk; + struct rtw89_phy_rfk_log_fmt *rfk_log_fmt; }; struct rtw89_fw_info { @@ -4010,6 +4073,17 @@ struct rtw89_sub_entity { struct rtw89_chanctx_cfg *cfg; }; +struct rtw89_edcca_bak { + u8 a; + u8 p; + u8 ppdu; + u8 th_old; +}; + +enum rtw89_dm_type { + RTW89_DM_DYNAMIC_EDCCA, +}; + struct rtw89_hal { u32 rx_fltr; u8 cv; @@ -4034,7 +4108,8 @@ struct rtw89_hal { bool entity_pause; enum rtw89_entity_mode entity_mode; - u32 edcca_bak; + struct rtw89_edcca_bak edcca_bak; + u32 disabled_dm_bitmap; /* bitmap of enum rtw89_dm_type */ }; #define RTW89_MAX_MAC_ID_NUM 128 @@ -4042,6 +4117,9 @@ struct rtw89_hal { enum rtw89_flags { RTW89_FLAG_POWERON, + RTW89_FLAG_DMAC_FUNC, + RTW89_FLAG_CMAC0_FUNC, + RTW89_FLAG_CMAC1_FUNC, RTW89_FLAG_FW_RDY, RTW89_FLAG_RUNNING, RTW89_FLAG_BFEE_MON, diff --git a/drivers/net/wireless/realtek/rtw89/debug.c b/drivers/net/wireless/realtek/rtw89/debug.c index a3f795d240ea5e..44829a1481859b 100644 --- a/drivers/net/wireless/realtek/rtw89/debug.c +++ b/drivers/net/wireless/realtek/rtw89/debug.c @@ -3330,13 +3330,14 @@ rtw89_debug_priv_early_h2c_set(struct file *filp, const char __user *user_buf, static int rtw89_dbg_trigger_ctrl_error(struct rtw89_dev *rtwdev) { + const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def; struct rtw89_cpuio_ctrl ctrl_para = {0}; u16 pkt_id; int ret; rtw89_leave_ps_mode(rtwdev); - ret = rtw89_mac_dle_buf_req(rtwdev, 0x20, true, &pkt_id); + ret = mac->dle_buf_req(rtwdev, 0x20, true, &pkt_id); if (ret) return ret; @@ -3348,7 +3349,7 @@ static int rtw89_dbg_trigger_ctrl_error(struct rtw89_dev *rtwdev) ctrl_para.dst_pid = WDE_DLE_PORT_ID_WDRLS; ctrl_para.dst_qid = WDE_DLE_QUEID_NO_REPORT; - if (rtw89_mac_set_cpuio(rtwdev, &ctrl_para, true)) + if (mac->set_cpuio(rtwdev, &ctrl_para, true)) return -EFAULT; return 0; @@ -3770,6 +3771,58 @@ static int rtw89_debug_priv_stations_get(struct seq_file *m, void *v) return 0; } +#define DM_INFO(type) {RTW89_DM_ ## type, #type} + +static const struct rtw89_disabled_dm_info { + enum rtw89_dm_type type; + const char *name; +} rtw89_disabled_dm_infos[] = { + DM_INFO(DYNAMIC_EDCCA), +}; + +static int +rtw89_debug_priv_disable_dm_get(struct seq_file *m, void *v) +{ + struct rtw89_debugfs_priv *debugfs_priv = m->private; + struct rtw89_dev *rtwdev = debugfs_priv->rtwdev; + const struct rtw89_disabled_dm_info *info; + struct rtw89_hal *hal = &rtwdev->hal; + u32 disabled; + int i; + + seq_printf(m, "Disabled DM: 0x%x\n", hal->disabled_dm_bitmap); + + for (i = 0; i < ARRAY_SIZE(rtw89_disabled_dm_infos); i++) { + info = &rtw89_disabled_dm_infos[i]; + disabled = BIT(info->type) & hal->disabled_dm_bitmap; + + seq_printf(m, "[%d] %s: %c\n", info->type, info->name, + disabled ? 'X' : 'O'); + } + + return 0; +} + +static ssize_t +rtw89_debug_priv_disable_dm_set(struct file *filp, const char __user *user_buf, + size_t count, loff_t *loff) +{ + struct seq_file *m = (struct seq_file *)filp->private_data; + struct rtw89_debugfs_priv *debugfs_priv = m->private; + struct rtw89_dev *rtwdev = debugfs_priv->rtwdev; + struct rtw89_hal *hal = &rtwdev->hal; + u32 conf; + int ret; + + ret = kstrtou32_from_user(user_buf, count, 0, &conf); + if (ret) + return -EINVAL; + + hal->disabled_dm_bitmap = conf; + + return count; +} + static struct rtw89_debugfs_priv rtw89_debug_priv_read_reg = { .cb_read = rtw89_debug_priv_read_reg_get, .cb_write = rtw89_debug_priv_read_reg_select, @@ -3845,6 +3898,11 @@ static struct rtw89_debugfs_priv rtw89_debug_priv_stations = { .cb_read = rtw89_debug_priv_stations_get, }; +static struct rtw89_debugfs_priv rtw89_debug_priv_disable_dm = { + .cb_read = rtw89_debug_priv_disable_dm_get, + .cb_write = rtw89_debug_priv_disable_dm_set, +}; + #define rtw89_debugfs_add(name, mode, fopname, parent) \ do { \ rtw89_debug_priv_ ##name.rtwdev = rtwdev; \ @@ -3885,13 +3943,13 @@ void rtw89_debugfs_init(struct rtw89_dev *rtwdev) rtw89_debugfs_add_w(fw_log_manual); rtw89_debugfs_add_r(phy_info); rtw89_debugfs_add_r(stations); + rtw89_debugfs_add_rw(disable_dm); } #endif #ifdef CONFIG_RTW89_DEBUGMSG -void __rtw89_debug(struct rtw89_dev *rtwdev, - enum rtw89_debug_mask mask, - const char *fmt, ...) +void rtw89_debug(struct rtw89_dev *rtwdev, enum rtw89_debug_mask mask, + const char *fmt, ...) { struct va_format vaf = { .fmt = fmt, @@ -3907,5 +3965,5 @@ void __rtw89_debug(struct rtw89_dev *rtwdev, va_end(args); } -EXPORT_SYMBOL(__rtw89_debug); +EXPORT_SYMBOL(rtw89_debug); #endif diff --git a/drivers/net/wireless/realtek/rtw89/debug.h b/drivers/net/wireless/realtek/rtw89/debug.h index b663ee24555a15..800ea59873a1d8 100644 --- a/drivers/net/wireless/realtek/rtw89/debug.h +++ b/drivers/net/wireless/realtek/rtw89/debug.h @@ -30,6 +30,7 @@ enum rtw89_debug_mask { RTW89_DBG_UL_TB = BIT(19), RTW89_DBG_CHAN = BIT(20), RTW89_DBG_ACPI = BIT(21), + RTW89_DBG_EDCCA = BIT(22), RTW89_DBG_UNEXP = BIT(31), }; @@ -58,12 +59,10 @@ static inline void rtw89_debugfs_init(struct rtw89_dev *rtwdev) {} #ifdef CONFIG_RTW89_DEBUGMSG extern unsigned int rtw89_debug_mask; -#define rtw89_debug(rtwdev, a...) __rtw89_debug(rtwdev, ##a) __printf(3, 4) -void __rtw89_debug(struct rtw89_dev *rtwdev, - enum rtw89_debug_mask mask, - const char *fmt, ...); +void rtw89_debug(struct rtw89_dev *rtwdev, enum rtw89_debug_mask mask, + const char *fmt, ...); static inline void rtw89_hex_dump(struct rtw89_dev *rtwdev, enum rtw89_debug_mask mask, const char *prefix_str, @@ -74,6 +73,12 @@ static inline void rtw89_hex_dump(struct rtw89_dev *rtwdev, print_hex_dump_bytes(prefix_str, DUMP_PREFIX_OFFSET, buf, len); } + +static inline bool rtw89_debug_is_enabled(struct rtw89_dev *rtwdev, + enum rtw89_debug_mask mask) +{ + return !!(rtw89_debug_mask & mask); +} #else static inline void rtw89_debug(struct rtw89_dev *rtwdev, enum rtw89_debug_mask mask, @@ -82,6 +87,11 @@ static inline void rtw89_hex_dump(struct rtw89_dev *rtwdev, enum rtw89_debug_mask mask, const char *prefix_str, const void *buf, size_t len) {} +static inline bool rtw89_debug_is_enabled(struct rtw89_dev *rtwdev, + enum rtw89_debug_mask mask) +{ + return false; +} #endif #endif diff --git a/drivers/net/wireless/realtek/rtw89/fw.c b/drivers/net/wireless/realtek/rtw89/fw.c index a732c22a2d5492..09684cea9731c3 100644 --- a/drivers/net/wireless/realtek/rtw89/fw.c +++ b/drivers/net/wireless/realtek/rtw89/fw.c @@ -401,10 +401,14 @@ int __rtw89_fw_recognize_from_elm(struct rtw89_dev *rtwdev, const union rtw89_fw_element_arg arg) { enum rtw89_fw_type type = arg.fw_type; + struct rtw89_hal *hal = &rtwdev->hal; struct rtw89_fw_suit *fw_suit; + if (hal->cv != elm->u.bbmcu.cv) + return 1; /* ignore this element */ + fw_suit = rtw89_fw_suit_get(rtwdev, type); - fw_suit->data = elm->u.common.contents; + fw_suit->data = elm->u.bbmcu.contents; fw_suit->size = le32_to_cpu(elm->size); return rtw89_fw_update_ver(rtwdev, type, fw_suit); @@ -453,6 +457,7 @@ static const struct __fw_feat_cfg fw_feat_tbl[] = { __CFG_FW_FEAT(RTL8852C, ge, 0, 27, 36, 0, SCAN_OFFLOAD), __CFG_FW_FEAT(RTL8852C, ge, 0, 27, 40, 0, CRASH_TRIGGER), __CFG_FW_FEAT(RTL8852C, ge, 0, 27, 56, 10, BEACON_FILTER), + __CFG_FW_FEAT(RTL8922A, ge, 0, 34, 30, 0, CRASH_TRIGGER), }; static void rtw89_fw_iterate_feature_cfg(struct rtw89_fw_info *fw, @@ -658,6 +663,97 @@ int rtw89_fw_recognize_txpwr_from_elm(struct rtw89_dev *rtwdev, return 0; } +static +int rtw89_build_txpwr_trk_tbl_from_elm(struct rtw89_dev *rtwdev, + const struct rtw89_fw_element_hdr *elm, + const union rtw89_fw_element_arg arg) +{ + struct rtw89_fw_elm_info *elm_info = &rtwdev->fw.elm_info; + const struct rtw89_chip_info *chip = rtwdev->chip; + u32 needed_bitmap = 0; + u32 offset = 0; + int subband; + u32 bitmap; + int type; + + if (chip->support_bands & BIT(NL80211_BAND_6GHZ)) + needed_bitmap |= RTW89_DEFAULT_NEEDED_FW_TXPWR_TRK_6GHZ; + if (chip->support_bands & BIT(NL80211_BAND_5GHZ)) + needed_bitmap |= RTW89_DEFAULT_NEEDED_FW_TXPWR_TRK_5GHZ; + if (chip->support_bands & BIT(NL80211_BAND_2GHZ)) + needed_bitmap |= RTW89_DEFAULT_NEEDED_FW_TXPWR_TRK_2GHZ; + + bitmap = le32_to_cpu(elm->u.txpwr_trk.bitmap); + + if ((bitmap & needed_bitmap) != needed_bitmap) { + rtw89_warn(rtwdev, "needed txpwr trk bitmap %08x but %0x8x\n", + needed_bitmap, bitmap); + return -ENOENT; + } + + elm_info->txpwr_trk = kzalloc(sizeof(*elm_info->txpwr_trk), GFP_KERNEL); + if (!elm_info->txpwr_trk) + return -ENOMEM; + + for (type = 0; bitmap; type++, bitmap >>= 1) { + if (!(bitmap & BIT(0))) + continue; + + if (type >= __RTW89_FW_TXPWR_TRK_TYPE_6GHZ_START && + type <= __RTW89_FW_TXPWR_TRK_TYPE_6GHZ_MAX) + subband = 4; + else if (type >= __RTW89_FW_TXPWR_TRK_TYPE_5GHZ_START && + type <= __RTW89_FW_TXPWR_TRK_TYPE_5GHZ_MAX) + subband = 3; + else if (type >= __RTW89_FW_TXPWR_TRK_TYPE_2GHZ_START && + type <= __RTW89_FW_TXPWR_TRK_TYPE_2GHZ_MAX) + subband = 1; + else + break; + + elm_info->txpwr_trk->delta[type] = &elm->u.txpwr_trk.contents[offset]; + + offset += subband; + if (offset * DELTA_SWINGIDX_SIZE > le32_to_cpu(elm->size)) + goto err; + } + + return 0; + +err: + rtw89_warn(rtwdev, "unexpected txpwr trk offset %d over size %d\n", + offset, le32_to_cpu(elm->size)); + kfree(elm_info->txpwr_trk); + elm_info->txpwr_trk = NULL; + + return -EFAULT; +} + +static +int rtw89_build_rfk_log_fmt_from_elm(struct rtw89_dev *rtwdev, + const struct rtw89_fw_element_hdr *elm, + const union rtw89_fw_element_arg arg) +{ + struct rtw89_fw_elm_info *elm_info = &rtwdev->fw.elm_info; + u8 rfk_id; + + if (elm_info->rfk_log_fmt) + goto allocated; + + elm_info->rfk_log_fmt = kzalloc(sizeof(*elm_info->rfk_log_fmt), GFP_KERNEL); + if (!elm_info->rfk_log_fmt) + return 1; /* this is an optional element, so just ignore this */ + +allocated: + rfk_id = elm->u.rfk_log_fmt.rfk_id; + if (rfk_id >= RTW89_PHY_C2H_RFK_LOG_FUNC_NUM) + return 1; + + elm_info->rfk_log_fmt->elm[rfk_id] = elm; + + return 0; +} + static const struct rtw89_fw_element_handler __fw_element_handlers[] = { [RTW89_FW_ELEMENT_ID_BBMCU0] = {__rtw89_fw_recognize_from_elm, { .fw_type = RTW89_FW_BBMCU0 }, NULL}, @@ -710,6 +806,12 @@ static const struct rtw89_fw_element_handler __fw_element_handlers[] = { rtw89_fw_recognize_txpwr_from_elm, { .offset = offsetof(struct rtw89_rfe_data, tx_shape_lmt_ru.conf) }, NULL, }, + [RTW89_FW_ELEMENT_ID_TXPWR_TRK] = { + rtw89_build_txpwr_trk_tbl_from_elm, {}, "PWR_TRK", + }, + [RTW89_FW_ELEMENT_ID_RFKLOG_FMT] = { + rtw89_build_rfk_log_fmt_from_elm, {}, NULL, + }, }; int rtw89_fw_recognize_elements(struct rtw89_dev *rtwdev) @@ -750,6 +852,8 @@ int rtw89_fw_recognize_elements(struct rtw89_dev *rtwdev) goto next; ret = handler->fn(rtwdev, hdr, handler->arg); + if (ret == 1) /* ignore this element */ + goto next; if (ret) return ret; @@ -956,16 +1060,24 @@ static int rtw89_fw_download_main(struct rtw89_dev *rtwdev, static void rtw89_fw_prog_cnt_dump(struct rtw89_dev *rtwdev) { + enum rtw89_chip_gen chip_gen = rtwdev->chip->chip_gen; + u32 addr = R_AX_DBG_PORT_SEL; u32 val32; u16 index; + if (chip_gen == RTW89_CHIP_BE) { + addr = R_BE_WLCPU_PORT_PC; + goto dump; + } + rtw89_write32(rtwdev, R_AX_DBG_CTRL, FIELD_PREP(B_AX_DBG_SEL0, FW_PROG_CNTR_DBG_SEL) | FIELD_PREP(B_AX_DBG_SEL1, FW_PROG_CNTR_DBG_SEL)); rtw89_write32_mask(rtwdev, R_AX_SYS_STATUS1, B_AX_SEL_0XC0_MASK, MAC_DBG_SEL); +dump: for (index = 0; index < 15; index++) { - val32 = rtw89_read32(rtwdev, R_AX_DBG_PORT_SEL); + val32 = rtw89_read32(rtwdev, addr); rtw89_err(rtwdev, "[ERR]fw PC = 0x%x\n", val32); fsleep(10); } @@ -1135,6 +1247,9 @@ static void rtw89_unload_firmware_elements(struct rtw89_dev *rtwdev) for (i = 0; i < ARRAY_SIZE(elm_info->rf_radio); i++) rtw89_free_phy_tbl_from_elm(elm_info->rf_radio[i]); rtw89_free_phy_tbl_from_elm(elm_info->rf_nctl); + + kfree(elm_info->txpwr_trk); + kfree(elm_info->rfk_log_fmt); } void rtw89_unload_firmware(struct rtw89_dev *rtwdev) @@ -2215,6 +2330,41 @@ int rtw89_fw_h2c_join_info(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif, return ret; } +int rtw89_fw_h2c_notify_dbcc(struct rtw89_dev *rtwdev, bool en) +{ + struct rtw89_h2c_notify_dbcc *h2c; + u32 len = sizeof(*h2c); + struct sk_buff *skb; + int ret; + + skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, len); + if (!skb) { + rtw89_err(rtwdev, "failed to alloc skb for h2c notify dbcc\n"); + return -ENOMEM; + } + skb_put(skb, len); + h2c = (struct rtw89_h2c_notify_dbcc *)skb->data; + + h2c->w0 = le32_encode_bits(en, RTW89_H2C_NOTIFY_DBCC_EN); + + rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C, + H2C_CAT_MAC, H2C_CL_MAC_MEDIA_RPT, + H2C_FUNC_NOTIFY_DBCC, 0, 1, + len); + + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { + rtw89_err(rtwdev, "failed to send h2c\n"); + goto fail; + } + + return 0; +fail: + dev_kfree_skb_any(skb); + + return ret; +} + int rtw89_fw_h2c_macid_pause(struct rtw89_dev *rtwdev, u8 sh, u8 grp, bool pause) { @@ -3451,6 +3601,8 @@ static bool rtw89_fw_c2h_chk_atomic(struct rtw89_dev *rtwdev, return false; case RTW89_C2H_CAT_MAC: return rtw89_mac_c2h_chk_atomic(rtwdev, class, func); + case RTW89_C2H_CAT_OUTSRC: + return rtw89_phy_c2h_chk_atomic(rtwdev, class, func); } } @@ -3867,6 +4019,8 @@ static void rtw89_hw_scan_add_chan(struct rtw89_dev *rtwdev, int chan_type, if (info->channel_6ghz && ch_info->pri_ch != info->channel_6ghz) continue; + else if (info->channel_6ghz && probe_count != 0) + ch_info->period += RTW89_CHANNEL_TIME_6G; ch_info->pkt_id[probe_count++] = info->id; if (probe_count >= RTW89_SCANOFLD_MAX_SSID) break; @@ -4043,6 +4197,7 @@ void rtw89_hw_scan_complete(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif, rtw89_core_scan_complete(rtwdev, vif, true); ieee80211_scan_completed(rtwdev->hw, &info); ieee80211_wake_queues(rtwdev->hw); + rtw89_mac_enable_beacon_for_ap_vifs(rtwdev, true); rtw89_release_pkt_list(rtwdev); rtwvif = (struct rtw89_vif *)vif->drv_priv; @@ -4060,6 +4215,19 @@ void rtw89_hw_scan_abort(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif) rtw89_hw_scan_complete(rtwdev, vif, true); } +static bool rtw89_is_any_vif_connected_or_connecting(struct rtw89_dev *rtwdev) +{ + struct rtw89_vif *rtwvif; + + rtw89_for_each_rtwvif(rtwdev, rtwvif) { + /* This variable implies connected or during attempt to connect */ + if (!is_zero_ether_addr(rtwvif->bssid)) + return true; + } + + return false; +} + int rtw89_hw_scan_offload(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif, bool enable) { @@ -4072,8 +4240,7 @@ int rtw89_hw_scan_offload(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif, if (!rtwvif) return -EINVAL; - /* This variable implies connected or during attempt to connect */ - connected = !is_zero_ether_addr(rtwvif->bssid); + connected = rtw89_is_any_vif_connected_or_connecting(rtwdev); opt.enable = enable; opt.target_ch_mode = connected; if (enable) { diff --git a/drivers/net/wireless/realtek/rtw89/fw.h b/drivers/net/wireless/realtek/rtw89/fw.h index d4db9ab0b5e82c..01016588b1fcb4 100644 --- a/drivers/net/wireless/realtek/rtw89/fw.h +++ b/drivers/net/wireless/realtek/rtw89/fw.h @@ -1685,6 +1685,12 @@ static inline void SET_JOININFO_SELF_ROLE(void *h2c, u32 val) le32p_replace_bits((__le32 *)h2c, val, GENMASK(31, 30)); } +struct rtw89_h2c_notify_dbcc { + __le32 w0; +} __packed; + +#define RTW89_H2C_NOTIFY_DBCC_EN BIT(0) + static inline void SET_GENERAL_PKT_MACID(void *h2c, u32 val) { le32p_replace_bits((__le32 *)h2c, val, GENMASK(7, 0)); @@ -3426,6 +3432,8 @@ enum rtw89_fw_element_id { RTW89_FW_ELEMENT_ID_TXPWR_LMT_RU_6GHZ = 15, RTW89_FW_ELEMENT_ID_TX_SHAPE_LMT = 16, RTW89_FW_ELEMENT_ID_TX_SHAPE_LMT_RU = 17, + RTW89_FW_ELEMENT_ID_TXPWR_TRK = 18, + RTW89_FW_ELEMENT_ID_RFKLOG_FMT = 19, RTW89_FW_ELEMENT_ID_NUM, }; @@ -3446,6 +3454,7 @@ enum rtw89_fw_element_id { BIT(RTW89_FW_ELEMENT_ID_RADIO_A) | \ BIT(RTW89_FW_ELEMENT_ID_RADIO_B) | \ BIT(RTW89_FW_ELEMENT_ID_RF_NCTL) | \ + BIT(RTW89_FW_ELEMENT_ID_TXPWR_TRK) | \ BITS_OF_RTW89_TXPWR_FW_ELEMENTS) struct __rtw89_fw_txpwr_element { @@ -3457,6 +3466,59 @@ struct __rtw89_fw_txpwr_element { u8 content[]; } __packed; +enum rtw89_fw_txpwr_trk_type { + __RTW89_FW_TXPWR_TRK_TYPE_6GHZ_START = 0, + RTW89_FW_TXPWR_TRK_TYPE_6GB_N = 0, + RTW89_FW_TXPWR_TRK_TYPE_6GB_P = 1, + RTW89_FW_TXPWR_TRK_TYPE_6GA_N = 2, + RTW89_FW_TXPWR_TRK_TYPE_6GA_P = 3, + __RTW89_FW_TXPWR_TRK_TYPE_6GHZ_MAX = 3, + + __RTW89_FW_TXPWR_TRK_TYPE_5GHZ_START = 4, + RTW89_FW_TXPWR_TRK_TYPE_5GB_N = 4, + RTW89_FW_TXPWR_TRK_TYPE_5GB_P = 5, + RTW89_FW_TXPWR_TRK_TYPE_5GA_N = 6, + RTW89_FW_TXPWR_TRK_TYPE_5GA_P = 7, + __RTW89_FW_TXPWR_TRK_TYPE_5GHZ_MAX = 7, + + __RTW89_FW_TXPWR_TRK_TYPE_2GHZ_START = 8, + RTW89_FW_TXPWR_TRK_TYPE_2GB_N = 8, + RTW89_FW_TXPWR_TRK_TYPE_2GB_P = 9, + RTW89_FW_TXPWR_TRK_TYPE_2GA_N = 10, + RTW89_FW_TXPWR_TRK_TYPE_2GA_P = 11, + RTW89_FW_TXPWR_TRK_TYPE_2G_CCK_B_N = 12, + RTW89_FW_TXPWR_TRK_TYPE_2G_CCK_B_P = 13, + RTW89_FW_TXPWR_TRK_TYPE_2G_CCK_A_N = 14, + RTW89_FW_TXPWR_TRK_TYPE_2G_CCK_A_P = 15, + __RTW89_FW_TXPWR_TRK_TYPE_2GHZ_MAX = 15, + + RTW89_FW_TXPWR_TRK_TYPE_NR, +}; + +struct rtw89_fw_txpwr_track_cfg { + const s8 (*delta[RTW89_FW_TXPWR_TRK_TYPE_NR])[DELTA_SWINGIDX_SIZE]; +}; + +#define RTW89_DEFAULT_NEEDED_FW_TXPWR_TRK_6GHZ \ + (BIT(RTW89_FW_TXPWR_TRK_TYPE_6GB_N) | \ + BIT(RTW89_FW_TXPWR_TRK_TYPE_6GB_P) | \ + BIT(RTW89_FW_TXPWR_TRK_TYPE_6GA_N) | \ + BIT(RTW89_FW_TXPWR_TRK_TYPE_6GA_P)) +#define RTW89_DEFAULT_NEEDED_FW_TXPWR_TRK_5GHZ \ + (BIT(RTW89_FW_TXPWR_TRK_TYPE_5GB_N) | \ + BIT(RTW89_FW_TXPWR_TRK_TYPE_5GB_P) | \ + BIT(RTW89_FW_TXPWR_TRK_TYPE_5GA_N) | \ + BIT(RTW89_FW_TXPWR_TRK_TYPE_5GA_P)) +#define RTW89_DEFAULT_NEEDED_FW_TXPWR_TRK_2GHZ \ + (BIT(RTW89_FW_TXPWR_TRK_TYPE_2GB_N) | \ + BIT(RTW89_FW_TXPWR_TRK_TYPE_2GB_P) | \ + BIT(RTW89_FW_TXPWR_TRK_TYPE_2GA_N) | \ + BIT(RTW89_FW_TXPWR_TRK_TYPE_2GA_P) | \ + BIT(RTW89_FW_TXPWR_TRK_TYPE_2G_CCK_B_N) | \ + BIT(RTW89_FW_TXPWR_TRK_TYPE_2G_CCK_B_P) | \ + BIT(RTW89_FW_TXPWR_TRK_TYPE_2G_CCK_A_N) | \ + BIT(RTW89_FW_TXPWR_TRK_TYPE_2G_CCK_A_P)) + struct rtw89_fw_element_hdr { __le32 id; /* enum rtw89_fw_element_id */ __le32 size; /* exclude header size */ @@ -3477,6 +3539,23 @@ struct rtw89_fw_element_hdr { __le32 data; } __packed regs[]; } __packed reg2; + struct { + u8 cv; + u8 priv[7]; + u8 contents[]; + } __packed bbmcu; + struct { + __le32 bitmap; /* bitmap of enum rtw89_fw_txpwr_trk_type */ + __le32 rsvd; + s8 contents[][DELTA_SWINGIDX_SIZE]; + } __packed txpwr_trk; + struct { + u8 nr; + u8 rsvd[3]; + u8 rfk_id; /* enum rtw89_phy_c2h_rfk_log_func */ + u8 rsvd1[3]; + __le16 offset[]; + } __packed rfk_log_fmt; struct __rtw89_fw_txpwr_element txpwr; } __packed u; } __packed; @@ -3577,6 +3656,7 @@ struct rtw89_fw_h2c_rf_reg_info { #define H2C_CL_MAC_MEDIA_RPT 0x8 #define H2C_FUNC_MAC_JOININFO 0x0 #define H2C_FUNC_MAC_FWROLE_MAINTAIN 0x4 +#define H2C_FUNC_NOTIFY_DBCC 0x5 /* CLASS 9 - FW offload */ #define H2C_CL_MAC_FW_OFLD 0x9 @@ -3649,9 +3729,78 @@ struct rtw89_fw_h2c_rf_get_mccch { __le32 current_band_type; } __packed; -#define RTW89_FW_RSVD_PLE_SIZE 0x800 +enum rtw89_rf_log_type { + RTW89_RF_RUN_LOG = 0, + RTW89_RF_RPT_LOG = 1, +}; -#define RTW89_WCPU_BASE_MASK GENMASK(27, 0) +struct rtw89_c2h_rf_log_hdr { + u8 type; /* enum rtw89_rf_log_type */ + __le16 len; + u8 content[]; +} __packed; + +struct rtw89_c2h_rf_run_log { + __le32 fmt_idx; + __le32 arg[4]; +} __packed; + +struct rtw89_c2h_rf_dpk_rpt_log { + u8 ver; + u8 idx[2]; + u8 band[2]; + u8 bw[2]; + u8 ch[2]; + u8 path_ok[2]; + u8 txagc[2]; + u8 ther[2]; + u8 gs[2]; + u8 dc_i[4]; + u8 dc_q[4]; + u8 corr_val[2]; + u8 corr_idx[2]; + u8 is_timeout[2]; + u8 rxbb_ov[2]; + u8 rsvd; +} __packed; + +struct rtw89_c2h_rf_dack_rpt_log { + u8 fwdack_ver; + u8 fwdack_rpt_ver; + u8 msbk_d[2][2][16]; + u8 dadck_d[2][2]; + u8 cdack_d[2][2][2]; + __le16 addck2_d[2][2][2]; + u8 adgaink_d[2][2]; + __le16 biask_d[2][2]; + u8 addck_timeout; + u8 cdack_timeout; + u8 dadck_timeout; + u8 msbk_timeout; + u8 adgaink_timeout; + u8 dack_fail; +} __packed; + +struct rtw89_c2h_rf_rxdck_rpt_log { + u8 ver; + u8 band[2]; + u8 bw[2]; + u8 ch[2]; + u8 timeout[2]; +} __packed; + +struct rtw89_c2h_rf_txgapk_rpt_log { + __le32 r0x8010[2]; + __le32 chk_cnt; + u8 track_d[2][17]; + u8 power_d[2][17]; + u8 is_txgapk_ok; + u8 chk_id; + u8 ver; + u8 rsv1; +} __packed; + +#define RTW89_FW_RSVD_PLE_SIZE 0x800 #define RTW89_FW_BACKTRACE_INFO_SIZE 8 #define RTW89_VALID_FW_BACKTRACE_SIZE(_size) \ @@ -3704,6 +3853,7 @@ int rtw89_fw_h2c_role_maintain(struct rtw89_dev *rtwdev, enum rtw89_upd_mode upd_mode); int rtw89_fw_h2c_join_info(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif, struct rtw89_sta *rtwsta, bool dis_conn); +int rtw89_fw_h2c_notify_dbcc(struct rtw89_dev *rtwdev, bool en); int rtw89_fw_h2c_macid_pause(struct rtw89_dev *rtwdev, u8 sh, u8 grp, bool pause); int rtw89_fw_h2c_set_edca(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif, diff --git a/drivers/net/wireless/realtek/rtw89/mac.c b/drivers/net/wireless/realtek/rtw89/mac.c index 9ec3070516115c..80ab5fdfdaa0b5 100644 --- a/drivers/net/wireless/realtek/rtw89/mac.c +++ b/drivers/net/wireless/realtek/rtw89/mac.c @@ -57,8 +57,8 @@ static u32 rtw89_mac_mem_read(struct rtw89_dev *rtwdev, u32 offset, return rtw89_read32(rtwdev, mac->indir_access_addr); } -int rtw89_mac_check_mac_en(struct rtw89_dev *rtwdev, u8 mac_idx, - enum rtw89_mac_hwmod_sel sel) +static int rtw89_mac_check_mac_en_ax(struct rtw89_dev *rtwdev, u8 mac_idx, + enum rtw89_mac_hwmod_sel sel) { u32 val, r_val; @@ -113,8 +113,7 @@ int rtw89_mac_read_lte(struct rtw89_dev *rtwdev, const u32 offset, u32 *val) return ret; } -static -int dle_dfi_ctrl(struct rtw89_dev *rtwdev, struct rtw89_mac_dle_dfi_ctrl *ctrl) +int rtw89_mac_dle_dfi_cfg(struct rtw89_dev *rtwdev, struct rtw89_mac_dle_dfi_ctrl *ctrl) { u32 ctrl_reg, data_reg, ctrl_data; u32 val; @@ -154,8 +153,8 @@ int dle_dfi_ctrl(struct rtw89_dev *rtwdev, struct rtw89_mac_dle_dfi_ctrl *ctrl) return 0; } -static int dle_dfi_quota(struct rtw89_dev *rtwdev, - struct rtw89_mac_dle_dfi_quota *quota) +int rtw89_mac_dle_dfi_quota_cfg(struct rtw89_dev *rtwdev, + struct rtw89_mac_dle_dfi_quota *quota) { struct rtw89_mac_dle_dfi_ctrl ctrl; int ret; @@ -163,9 +162,9 @@ static int dle_dfi_quota(struct rtw89_dev *rtwdev, ctrl.type = quota->dle_type; ctrl.target = DLE_DFI_TYPE_QUOTA; ctrl.addr = quota->qtaid; - ret = dle_dfi_ctrl(rtwdev, &ctrl); + ret = rtw89_mac_dle_dfi_cfg(rtwdev, &ctrl); if (ret) { - rtw89_warn(rtwdev, "[ERR]dle_dfi_ctrl %d\n", ret); + rtw89_warn(rtwdev, "[ERR] dle dfi quota %d\n", ret); return ret; } @@ -174,8 +173,8 @@ static int dle_dfi_quota(struct rtw89_dev *rtwdev, return 0; } -static int dle_dfi_qempty(struct rtw89_dev *rtwdev, - struct rtw89_mac_dle_dfi_qempty *qempty) +int rtw89_mac_dle_dfi_qempty_cfg(struct rtw89_dev *rtwdev, + struct rtw89_mac_dle_dfi_qempty *qempty) { struct rtw89_mac_dle_dfi_ctrl ctrl; u32 ret; @@ -183,9 +182,9 @@ static int dle_dfi_qempty(struct rtw89_dev *rtwdev, ctrl.type = qempty->dle_type; ctrl.target = DLE_DFI_TYPE_QEMPTY; ctrl.addr = qempty->grpsel; - ret = dle_dfi_ctrl(rtwdev, &ctrl); + ret = rtw89_mac_dle_dfi_cfg(rtwdev, &ctrl); if (ret) { - rtw89_warn(rtwdev, "[ERR]dle_dfi_ctrl %d\n", ret); + rtw89_warn(rtwdev, "[ERR] dle dfi qempty %d\n", ret); return ret; } @@ -193,7 +192,7 @@ static int dle_dfi_qempty(struct rtw89_dev *rtwdev, return 0; } -static void dump_err_status_dispatcher(struct rtw89_dev *rtwdev) +static void dump_err_status_dispatcher_ax(struct rtw89_dev *rtwdev) { rtw89_info(rtwdev, "R_AX_HOST_DISPATCHER_ALWAYS_IMR=0x%08x ", rtw89_read32(rtwdev, R_AX_HOST_DISPATCHER_ERR_IMR)); @@ -209,7 +208,7 @@ static void dump_err_status_dispatcher(struct rtw89_dev *rtwdev) rtw89_read32(rtwdev, R_AX_OTHER_DISPATCHER_ERR_ISR)); } -static void rtw89_mac_dump_qta_lost(struct rtw89_dev *rtwdev) +static void rtw89_mac_dump_qta_lost_ax(struct rtw89_dev *rtwdev) { struct rtw89_mac_dle_dfi_qempty qempty; struct rtw89_mac_dle_dfi_quota quota; @@ -220,7 +219,7 @@ static void rtw89_mac_dump_qta_lost(struct rtw89_dev *rtwdev) qempty.dle_type = DLE_CTRL_TYPE_PLE; qempty.grpsel = 0; qempty.qempty = ~(u32)0; - ret = dle_dfi_qempty(rtwdev, &qempty); + ret = rtw89_mac_dle_dfi_qempty_cfg(rtwdev, &qempty); if (ret) rtw89_warn(rtwdev, "%s: query DLE fail\n", __func__); else @@ -232,19 +231,19 @@ static void rtw89_mac_dump_qta_lost(struct rtw89_dev *rtwdev) ctrl.type = DLE_CTRL_TYPE_PLE; ctrl.target = DLE_DFI_TYPE_QLNKTBL; ctrl.addr = (QLNKTBL_ADDR_INFO_SEL_0 ? QLNKTBL_ADDR_INFO_SEL : 0) | - FIELD_PREP(QLNKTBL_ADDR_TBL_IDX_MASK, i); - ret = dle_dfi_ctrl(rtwdev, &ctrl); + u32_encode_bits(i, QLNKTBL_ADDR_TBL_IDX_MASK); + ret = rtw89_mac_dle_dfi_cfg(rtwdev, &ctrl); if (ret) rtw89_warn(rtwdev, "%s: query DLE fail\n", __func__); else - rtw89_info(rtwdev, "qidx%d pktcnt = %ld\n", i, - FIELD_GET(QLNKTBL_DATA_SEL1_PKT_CNT_MASK, - ctrl.out_data)); + rtw89_info(rtwdev, "qidx%d pktcnt = %d\n", i, + u32_get_bits(ctrl.out_data, + QLNKTBL_DATA_SEL1_PKT_CNT_MASK)); } quota.dle_type = DLE_CTRL_TYPE_PLE; quota.qtaid = 6; - ret = dle_dfi_quota(rtwdev, "a); + ret = rtw89_mac_dle_dfi_quota_cfg(rtwdev, "a); if (ret) rtw89_warn(rtwdev, "%s: query DLE fail\n", __func__); else @@ -252,33 +251,74 @@ static void rtw89_mac_dump_qta_lost(struct rtw89_dev *rtwdev) quota.rsv_pgnum, quota.use_pgnum); val = rtw89_read32(rtwdev, R_AX_PLE_QTA6_CFG); - rtw89_info(rtwdev, "[PLE][CMAC0_RX]min_pgnum=0x%lx\n", - FIELD_GET(B_AX_PLE_Q6_MIN_SIZE_MASK, val)); - rtw89_info(rtwdev, "[PLE][CMAC0_RX]max_pgnum=0x%lx\n", - FIELD_GET(B_AX_PLE_Q6_MAX_SIZE_MASK, val)); + rtw89_info(rtwdev, "[PLE][CMAC0_RX]min_pgnum=0x%x\n", + u32_get_bits(val, B_AX_PLE_Q6_MIN_SIZE_MASK)); + rtw89_info(rtwdev, "[PLE][CMAC0_RX]max_pgnum=0x%x\n", + u32_get_bits(val, B_AX_PLE_Q6_MAX_SIZE_MASK)); + val = rtw89_read32(rtwdev, R_AX_RX_FLTR_OPT); + rtw89_info(rtwdev, "[PLE][CMAC0_RX]B_AX_RX_MPDU_MAX_LEN=0x%x\n", + u32_get_bits(val, B_AX_RX_MPDU_MAX_LEN_MASK)); + rtw89_info(rtwdev, "R_AX_RSP_CHK_SIG=0x%08x\n", + rtw89_read32(rtwdev, R_AX_RSP_CHK_SIG)); + rtw89_info(rtwdev, "R_AX_TRXPTCL_RESP_0=0x%08x\n", + rtw89_read32(rtwdev, R_AX_TRXPTCL_RESP_0)); + rtw89_info(rtwdev, "R_AX_CCA_CONTROL=0x%08x\n", + rtw89_read32(rtwdev, R_AX_CCA_CONTROL)); + + if (!rtw89_mac_check_mac_en(rtwdev, RTW89_MAC_1, RTW89_CMAC_SEL)) { + quota.dle_type = DLE_CTRL_TYPE_PLE; + quota.qtaid = 7; + ret = rtw89_mac_dle_dfi_quota_cfg(rtwdev, "a); + if (ret) + rtw89_warn(rtwdev, "%s: query DLE fail\n", __func__); + else + rtw89_info(rtwdev, "quota7 rsv/use: 0x%x/0x%x\n", + quota.rsv_pgnum, quota.use_pgnum); + + val = rtw89_read32(rtwdev, R_AX_PLE_QTA7_CFG); + rtw89_info(rtwdev, "[PLE][CMAC1_RX]min_pgnum=0x%x\n", + u32_get_bits(val, B_AX_PLE_Q7_MIN_SIZE_MASK)); + rtw89_info(rtwdev, "[PLE][CMAC1_RX]max_pgnum=0x%x\n", + u32_get_bits(val, B_AX_PLE_Q7_MAX_SIZE_MASK)); + val = rtw89_read32(rtwdev, R_AX_RX_FLTR_OPT_C1); + rtw89_info(rtwdev, "[PLE][CMAC1_RX]B_AX_RX_MPDU_MAX_LEN=0x%x\n", + u32_get_bits(val, B_AX_RX_MPDU_MAX_LEN_MASK)); + rtw89_info(rtwdev, "R_AX_RSP_CHK_SIG_C1=0x%08x\n", + rtw89_read32(rtwdev, R_AX_RSP_CHK_SIG_C1)); + rtw89_info(rtwdev, "R_AX_TRXPTCL_RESP_0_C1=0x%08x\n", + rtw89_read32(rtwdev, R_AX_TRXPTCL_RESP_0_C1)); + rtw89_info(rtwdev, "R_AX_CCA_CONTROL_C1=0x%08x\n", + rtw89_read32(rtwdev, R_AX_CCA_CONTROL_C1)); + } + + rtw89_info(rtwdev, "R_AX_DLE_EMPTY0=0x%08x\n", + rtw89_read32(rtwdev, R_AX_DLE_EMPTY0)); + rtw89_info(rtwdev, "R_AX_DLE_EMPTY1=0x%08x\n", + rtw89_read32(rtwdev, R_AX_DLE_EMPTY1)); - dump_err_status_dispatcher(rtwdev); + dump_err_status_dispatcher_ax(rtwdev); } -static void rtw89_mac_dump_l0_to_l1(struct rtw89_dev *rtwdev, - enum mac_ax_err_info err) +void rtw89_mac_dump_l0_to_l1(struct rtw89_dev *rtwdev, + enum mac_ax_err_info err) { + const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def; u32 dbg, event; dbg = rtw89_read32(rtwdev, R_AX_SER_DBG_INFO); - event = FIELD_GET(B_AX_L0_TO_L1_EVENT_MASK, dbg); + event = u32_get_bits(dbg, B_AX_L0_TO_L1_EVENT_MASK); switch (event) { case MAC_AX_L0_TO_L1_RX_QTA_LOST: rtw89_info(rtwdev, "quota lost!\n"); - rtw89_mac_dump_qta_lost(rtwdev); + mac->dump_qta_lost(rtwdev); break; default: break; } } -static void rtw89_mac_dump_dmac_err_status(struct rtw89_dev *rtwdev) +void rtw89_mac_dump_dmac_err_status(struct rtw89_dev *rtwdev) { const struct rtw89_chip_info *chip = rtwdev->chip; u32 dmac_err; @@ -358,6 +398,21 @@ static void rtw89_mac_dump_dmac_err_status(struct rtw89_dev *rtwdev) rtw89_info(rtwdev, "sel=%x,R_AX_SEC_DEBUG2=0x%08x\n", i, rtw89_read32(rtwdev, R_AX_SEC_DEBUG2)); } + } else if (chip->chip_id == RTL8922A) { + rtw89_info(rtwdev, "R_BE_SEC_ERROR_FLAG=0x%08x\n", + rtw89_read32(rtwdev, R_BE_SEC_ERROR_FLAG)); + rtw89_info(rtwdev, "R_BE_SEC_ERROR_IMR=0x%08x\n", + rtw89_read32(rtwdev, R_BE_SEC_ERROR_IMR)); + rtw89_info(rtwdev, "R_BE_SEC_ENG_CTRL=0x%08x\n", + rtw89_read32(rtwdev, R_BE_SEC_ENG_CTRL)); + rtw89_info(rtwdev, "R_BE_SEC_MPDU_PROC=0x%08x\n", + rtw89_read32(rtwdev, R_BE_SEC_MPDU_PROC)); + rtw89_info(rtwdev, "R_BE_SEC_CAM_ACCESS=0x%08x\n", + rtw89_read32(rtwdev, R_BE_SEC_CAM_ACCESS)); + rtw89_info(rtwdev, "R_BE_SEC_CAM_RDATA=0x%08x\n", + rtw89_read32(rtwdev, R_BE_SEC_CAM_RDATA)); + rtw89_info(rtwdev, "R_BE_SEC_DEBUG2=0x%08x\n", + rtw89_read32(rtwdev, R_BE_SEC_DEBUG2)); } else { rtw89_info(rtwdev, "R_AX_SEC_ERR_IMR_ISR=0x%08x\n", rtw89_read32(rtwdev, R_AX_SEC_DEBUG)); @@ -394,10 +449,17 @@ static void rtw89_mac_dump_dmac_err_status(struct rtw89_dev *rtwdev) } if (dmac_err & B_AX_STA_SCHEDULER_ERR_FLAG) { - rtw89_info(rtwdev, "R_AX_STA_SCHEDULER_ERR_IMR=0x%08x\n", - rtw89_read32(rtwdev, R_AX_STA_SCHEDULER_ERR_IMR)); - rtw89_info(rtwdev, "R_AX_STA_SCHEDULER_ERR_ISR=0x%08x\n", - rtw89_read32(rtwdev, R_AX_STA_SCHEDULER_ERR_ISR)); + if (chip->chip_id == RTL8922A) { + rtw89_info(rtwdev, "R_BE_INTERRUPT_MASK_REG=0x%08x\n", + rtw89_read32(rtwdev, R_BE_INTERRUPT_MASK_REG)); + rtw89_info(rtwdev, "R_BE_INTERRUPT_STS_REG=0x%08x\n", + rtw89_read32(rtwdev, R_BE_INTERRUPT_STS_REG)); + } else { + rtw89_info(rtwdev, "R_AX_STA_SCHEDULER_ERR_IMR=0x%08x\n", + rtw89_read32(rtwdev, R_AX_STA_SCHEDULER_ERR_IMR)); + rtw89_info(rtwdev, "R_AX_STA_SCHEDULER_ERR_ISR=0x%08x\n", + rtw89_read32(rtwdev, R_AX_STA_SCHEDULER_ERR_ISR)); + } } if (dmac_err & B_AX_WDE_DLE_ERR_FLAG) { @@ -412,7 +474,7 @@ static void rtw89_mac_dump_dmac_err_status(struct rtw89_dev *rtwdev) } if (dmac_err & B_AX_TXPKTCTRL_ERR_FLAG) { - if (chip->chip_id == RTL8852C) { + if (chip->chip_id == RTL8852C || chip->chip_id == RTL8922A) { rtw89_info(rtwdev, "R_AX_TXPKTCTL_B0_ERRFLAG_IMR=0x%08x\n", rtw89_read32(rtwdev, R_AX_TXPKTCTL_B0_ERRFLAG_IMR)); rtw89_info(rtwdev, "R_AX_TXPKTCTL_B0_ERRFLAG_ISR=0x%08x\n", @@ -444,30 +506,41 @@ static void rtw89_mac_dump_dmac_err_status(struct rtw89_dev *rtwdev) rtw89_read32(rtwdev, R_AX_WD_CPUQ_OP_1)); rtw89_info(rtwdev, "R_AX_WD_CPUQ_OP_2=0x%08x\n", rtw89_read32(rtwdev, R_AX_WD_CPUQ_OP_2)); - rtw89_info(rtwdev, "R_AX_WD_CPUQ_OP_STATUS=0x%08x\n", - rtw89_read32(rtwdev, R_AX_WD_CPUQ_OP_STATUS)); rtw89_info(rtwdev, "R_AX_PL_CPUQ_OP_0=0x%08x\n", rtw89_read32(rtwdev, R_AX_PL_CPUQ_OP_0)); rtw89_info(rtwdev, "R_AX_PL_CPUQ_OP_1=0x%08x\n", rtw89_read32(rtwdev, R_AX_PL_CPUQ_OP_1)); rtw89_info(rtwdev, "R_AX_PL_CPUQ_OP_2=0x%08x\n", rtw89_read32(rtwdev, R_AX_PL_CPUQ_OP_2)); - rtw89_info(rtwdev, "R_AX_PL_CPUQ_OP_STATUS=0x%08x\n", - rtw89_read32(rtwdev, R_AX_PL_CPUQ_OP_STATUS)); - if (chip->chip_id == RTL8852C) { - rtw89_info(rtwdev, "R_AX_RX_CTRL0=0x%08x\n", - rtw89_read32(rtwdev, R_AX_RX_CTRL0)); - rtw89_info(rtwdev, "R_AX_RX_CTRL1=0x%08x\n", - rtw89_read32(rtwdev, R_AX_RX_CTRL1)); - rtw89_info(rtwdev, "R_AX_RX_CTRL2=0x%08x\n", - rtw89_read32(rtwdev, R_AX_RX_CTRL2)); + if (chip->chip_id == RTL8922A) { + rtw89_info(rtwdev, "R_BE_WD_CPUQ_OP_3=0x%08x\n", + rtw89_read32(rtwdev, R_BE_WD_CPUQ_OP_3)); + rtw89_info(rtwdev, "R_BE_WD_CPUQ_OP_STATUS=0x%08x\n", + rtw89_read32(rtwdev, R_BE_WD_CPUQ_OP_STATUS)); + rtw89_info(rtwdev, "R_BE_PLE_CPUQ_OP_3=0x%08x\n", + rtw89_read32(rtwdev, R_BE_PL_CPUQ_OP_3)); + rtw89_info(rtwdev, "R_BE_PL_CPUQ_OP_STATUS=0x%08x\n", + rtw89_read32(rtwdev, R_BE_PL_CPUQ_OP_STATUS)); } else { - rtw89_info(rtwdev, "R_AX_RXDMA_PKT_INFO_0=0x%08x\n", - rtw89_read32(rtwdev, R_AX_RXDMA_PKT_INFO_0)); - rtw89_info(rtwdev, "R_AX_RXDMA_PKT_INFO_1=0x%08x\n", - rtw89_read32(rtwdev, R_AX_RXDMA_PKT_INFO_1)); - rtw89_info(rtwdev, "R_AX_RXDMA_PKT_INFO_2=0x%08x\n", - rtw89_read32(rtwdev, R_AX_RXDMA_PKT_INFO_2)); + rtw89_info(rtwdev, "R_AX_WD_CPUQ_OP_STATUS=0x%08x\n", + rtw89_read32(rtwdev, R_AX_WD_CPUQ_OP_STATUS)); + rtw89_info(rtwdev, "R_AX_PL_CPUQ_OP_STATUS=0x%08x\n", + rtw89_read32(rtwdev, R_AX_PL_CPUQ_OP_STATUS)); + if (chip->chip_id == RTL8852C) { + rtw89_info(rtwdev, "R_AX_RX_CTRL0=0x%08x\n", + rtw89_read32(rtwdev, R_AX_RX_CTRL0)); + rtw89_info(rtwdev, "R_AX_RX_CTRL1=0x%08x\n", + rtw89_read32(rtwdev, R_AX_RX_CTRL1)); + rtw89_info(rtwdev, "R_AX_RX_CTRL2=0x%08x\n", + rtw89_read32(rtwdev, R_AX_RX_CTRL2)); + } else { + rtw89_info(rtwdev, "R_AX_RXDMA_PKT_INFO_0=0x%08x\n", + rtw89_read32(rtwdev, R_AX_RXDMA_PKT_INFO_0)); + rtw89_info(rtwdev, "R_AX_RXDMA_PKT_INFO_1=0x%08x\n", + rtw89_read32(rtwdev, R_AX_RXDMA_PKT_INFO_1)); + rtw89_info(rtwdev, "R_AX_RXDMA_PKT_INFO_2=0x%08x\n", + rtw89_read32(rtwdev, R_AX_RXDMA_PKT_INFO_2)); + } } } @@ -479,22 +552,37 @@ static void rtw89_mac_dump_dmac_err_status(struct rtw89_dev *rtwdev) } if (dmac_err & B_AX_DISPATCH_ERR_FLAG) { - rtw89_info(rtwdev, "R_AX_HOST_DISPATCHER_ERR_IMR=0x%08x\n", - rtw89_read32(rtwdev, R_AX_HOST_DISPATCHER_ERR_IMR)); - rtw89_info(rtwdev, "R_AX_HOST_DISPATCHER_ERR_ISR=0x%08x\n", - rtw89_read32(rtwdev, R_AX_HOST_DISPATCHER_ERR_ISR)); - rtw89_info(rtwdev, "R_AX_CPU_DISPATCHER_ERR_IMR=0x%08x\n", - rtw89_read32(rtwdev, R_AX_CPU_DISPATCHER_ERR_IMR)); - rtw89_info(rtwdev, "R_AX_CPU_DISPATCHER_ERR_ISR=0x%08x\n", - rtw89_read32(rtwdev, R_AX_CPU_DISPATCHER_ERR_ISR)); - rtw89_info(rtwdev, "R_AX_OTHER_DISPATCHER_ERR_IMR=0x%08x\n", - rtw89_read32(rtwdev, R_AX_OTHER_DISPATCHER_ERR_IMR)); - rtw89_info(rtwdev, "R_AX_OTHER_DISPATCHER_ERR_ISR=0x%08x\n", - rtw89_read32(rtwdev, R_AX_OTHER_DISPATCHER_ERR_ISR)); + if (chip->chip_id == RTL8922A) { + rtw89_info(rtwdev, "R_BE_DISP_HOST_IMR=0x%08x\n", + rtw89_read32(rtwdev, R_BE_DISP_HOST_IMR)); + rtw89_info(rtwdev, "R_BE_DISP_ERROR_ISR1=0x%08x\n", + rtw89_read32(rtwdev, R_BE_DISP_ERROR_ISR1)); + rtw89_info(rtwdev, "R_BE_DISP_CPU_IMR=0x%08x\n", + rtw89_read32(rtwdev, R_BE_DISP_CPU_IMR)); + rtw89_info(rtwdev, "R_BE_DISP_ERROR_ISR2=0x%08x\n", + rtw89_read32(rtwdev, R_BE_DISP_ERROR_ISR2)); + rtw89_info(rtwdev, "R_BE_DISP_OTHER_IMR=0x%08x\n", + rtw89_read32(rtwdev, R_BE_DISP_OTHER_IMR)); + rtw89_info(rtwdev, "R_BE_DISP_ERROR_ISR0=0x%08x\n", + rtw89_read32(rtwdev, R_BE_DISP_ERROR_ISR0)); + } else { + rtw89_info(rtwdev, "R_AX_HOST_DISPATCHER_ERR_IMR=0x%08x\n", + rtw89_read32(rtwdev, R_AX_HOST_DISPATCHER_ERR_IMR)); + rtw89_info(rtwdev, "R_AX_HOST_DISPATCHER_ERR_ISR=0x%08x\n", + rtw89_read32(rtwdev, R_AX_HOST_DISPATCHER_ERR_ISR)); + rtw89_info(rtwdev, "R_AX_CPU_DISPATCHER_ERR_IMR=0x%08x\n", + rtw89_read32(rtwdev, R_AX_CPU_DISPATCHER_ERR_IMR)); + rtw89_info(rtwdev, "R_AX_CPU_DISPATCHER_ERR_ISR=0x%08x\n", + rtw89_read32(rtwdev, R_AX_CPU_DISPATCHER_ERR_ISR)); + rtw89_info(rtwdev, "R_AX_OTHER_DISPATCHER_ERR_IMR=0x%08x\n", + rtw89_read32(rtwdev, R_AX_OTHER_DISPATCHER_ERR_IMR)); + rtw89_info(rtwdev, "R_AX_OTHER_DISPATCHER_ERR_ISR=0x%08x\n", + rtw89_read32(rtwdev, R_AX_OTHER_DISPATCHER_ERR_ISR)); + } } if (dmac_err & B_AX_BBRPT_ERR_FLAG) { - if (chip->chip_id == RTL8852C) { + if (chip->chip_id == RTL8852C || chip->chip_id == RTL8922A) { rtw89_info(rtwdev, "R_AX_BBRPT_COM_ERR_IMR=0x%08x\n", rtw89_read32(rtwdev, R_AX_BBRPT_COM_ERR_IMR)); rtw89_info(rtwdev, "R_AX_BBRPT_COM_ERR_ISR=0x%08x\n", @@ -519,18 +607,54 @@ static void rtw89_mac_dump_dmac_err_status(struct rtw89_dev *rtwdev) rtw89_info(rtwdev, "R_AX_BBRPT_DFS_ERR_ISR=0x%08x\n", rtw89_read32(rtwdev, R_AX_BBRPT_DFS_ERR_ISR)); } + if (chip->chip_id == RTL8922A) { + rtw89_info(rtwdev, "R_BE_LA_ERRFLAG_IMR=0x%08x\n", + rtw89_read32(rtwdev, R_BE_LA_ERRFLAG_IMR)); + rtw89_info(rtwdev, "R_BE_LA_ERRFLAG_ISR=0x%08x\n", + rtw89_read32(rtwdev, R_BE_LA_ERRFLAG_ISR)); + } + } + + if (dmac_err & B_AX_HAXIDMA_ERR_FLAG) { + if (chip->chip_id == RTL8922A) { + rtw89_info(rtwdev, "R_BE_HAXI_IDCT_MSK=0x%08x\n", + rtw89_read32(rtwdev, R_BE_HAXI_IDCT_MSK)); + rtw89_info(rtwdev, "R_BE_HAXI_IDCT=0x%08x\n", + rtw89_read32(rtwdev, R_BE_HAXI_IDCT)); + } else if (chip->chip_id == RTL8852C) { + rtw89_info(rtwdev, "R_AX_HAXIDMA_ERR_IMR=0x%08x\n", + rtw89_read32(rtwdev, R_AX_HAXI_IDCT_MSK)); + rtw89_info(rtwdev, "R_AX_HAXIDMA_ERR_ISR=0x%08x\n", + rtw89_read32(rtwdev, R_AX_HAXI_IDCT)); + } + } + + if (dmac_err & B_BE_P_AXIDMA_ERR_INT) { + rtw89_info(rtwdev, "R_BE_PL_AXIDMA_IDCT_MSK=0x%08x\n", + rtw89_mac_mem_read(rtwdev, R_BE_PL_AXIDMA_IDCT_MSK, + RTW89_MAC_MEM_AXIDMA)); + rtw89_info(rtwdev, "R_BE_PL_AXIDMA_IDCT=0x%08x\n", + rtw89_mac_mem_read(rtwdev, R_BE_PL_AXIDMA_IDCT, + RTW89_MAC_MEM_AXIDMA)); + } + + if (dmac_err & B_BE_MLO_ERR_INT) { + rtw89_info(rtwdev, "R_BE_MLO_ERR_IDCT_IMR=0x%08x\n", + rtw89_read32(rtwdev, R_BE_MLO_ERR_IDCT_IMR)); + rtw89_info(rtwdev, "R_BE_PKTIN_ERR_ISR=0x%08x\n", + rtw89_read32(rtwdev, R_BE_MLO_ERR_IDCT_ISR)); } - if (dmac_err & B_AX_HAXIDMA_ERR_FLAG && chip->chip_id == RTL8852C) { - rtw89_info(rtwdev, "R_AX_HAXIDMA_ERR_IMR=0x%08x\n", - rtw89_read32(rtwdev, R_AX_HAXI_IDCT_MSK)); - rtw89_info(rtwdev, "R_AX_HAXIDMA_ERR_ISR=0x%08x\n", - rtw89_read32(rtwdev, R_AX_HAXI_IDCT)); + if (dmac_err & B_BE_PLRLS_ERR_INT) { + rtw89_info(rtwdev, "R_BE_PLRLS_ERR_IMR=0x%08x\n", + rtw89_read32(rtwdev, R_BE_PLRLS_ERR_IMR)); + rtw89_info(rtwdev, "R_BE_PLRLS_ERR_ISR=0x%08x\n", + rtw89_read32(rtwdev, R_BE_PLRLS_ERR_ISR)); } } -static void rtw89_mac_dump_cmac_err_status(struct rtw89_dev *rtwdev, - u8 band) +static void rtw89_mac_dump_cmac_err_status_ax(struct rtw89_dev *rtwdev, + u8 band) { const struct rtw89_chip_info *chip = rtwdev->chip; u32 offset = 0; @@ -620,8 +744,8 @@ static void rtw89_mac_dump_cmac_err_status(struct rtw89_dev *rtwdev, rtw89_read32(rtwdev, R_AX_CMAC_ERR_IMR + offset)); } -static void rtw89_mac_dump_err_status(struct rtw89_dev *rtwdev, - enum mac_ax_err_info err) +static void rtw89_mac_dump_err_status_ax(struct rtw89_dev *rtwdev, + enum mac_ax_err_info err) { if (err != MAC_AX_ERR_L1_ERR_DMAC && err != MAC_AX_ERR_L0_PROMOTE_TO_L1 && @@ -633,11 +757,16 @@ static void rtw89_mac_dump_err_status(struct rtw89_dev *rtwdev, rtw89_info(rtwdev, "--->\nerr=0x%x\n", err); rtw89_info(rtwdev, "R_AX_SER_DBG_INFO =0x%08x\n", rtw89_read32(rtwdev, R_AX_SER_DBG_INFO)); + rtw89_info(rtwdev, "R_AX_SER_DBG_INFO =0x%08x\n", + rtw89_read32(rtwdev, R_AX_SER_DBG_INFO)); + rtw89_info(rtwdev, "DBG Counter 1 (R_AX_DRV_FW_HSK_4)=0x%08x\n", + rtw89_read32(rtwdev, R_AX_DRV_FW_HSK_4)); + rtw89_info(rtwdev, "DBG Counter 2 (R_AX_DRV_FW_HSK_5)=0x%08x\n", + rtw89_read32(rtwdev, R_AX_DRV_FW_HSK_5)); rtw89_mac_dump_dmac_err_status(rtwdev); - rtw89_mac_dump_cmac_err_status(rtwdev, RTW89_MAC_0); - if (rtwdev->dbcc_en) - rtw89_mac_dump_cmac_err_status(rtwdev, RTW89_MAC_1); + rtw89_mac_dump_cmac_err_status_ax(rtwdev, RTW89_MAC_0); + rtw89_mac_dump_cmac_err_status_ax(rtwdev, RTW89_MAC_1); rtwdev->hci.ops->dump_err_status(rtwdev); @@ -682,6 +811,7 @@ static bool rtw89_mac_suppress_log(struct rtw89_dev *rtwdev, u32 err) u32 rtw89_mac_get_err_status(struct rtw89_dev *rtwdev) { + const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def; u32 err, err_scnr; int ret; @@ -707,7 +837,7 @@ u32 rtw89_mac_get_err_status(struct rtw89_dev *rtwdev) return err; rtw89_fw_st_dbg_dump(rtwdev); - rtw89_mac_dump_err_status(rtwdev, err); + mac->dump_err_status(rtwdev, err); return err; } @@ -901,7 +1031,7 @@ static int hfc_pub_ctrl(struct rtw89_dev *rtwdev) return 0; } -static int hfc_upd_mix_info(struct rtw89_dev *rtwdev) +static void hfc_get_mix_info_ax(struct rtw89_dev *rtwdev) { const struct rtw89_chip_info *chip = rtwdev->chip; const struct rtw89_page_regs *regs = chip->page_regs; @@ -910,11 +1040,6 @@ static int hfc_upd_mix_info(struct rtw89_dev *rtwdev) struct rtw89_hfc_prec_cfg *prec_cfg = ¶m->prec_cfg; struct rtw89_hfc_pub_info *info = ¶m->pub_info; u32 val; - int ret; - - ret = rtw89_mac_check_mac_en(rtwdev, RTW89_MAC_0, RTW89_DMAC_SEL); - if (ret) - return ret; val = rtw89_read32(rtwdev, regs->pub_page_info1); info->g0_used = u32_get_bits(val, B_AX_G0_USE_PG_MASK); @@ -959,6 +1084,19 @@ static int hfc_upd_mix_info(struct rtw89_dev *rtwdev) val = rtw89_read32(rtwdev, regs->pub_page_ctrl1); pub_cfg->grp0 = u32_get_bits(val, B_AX_PUBPG_G0_MASK); pub_cfg->grp1 = u32_get_bits(val, B_AX_PUBPG_G1_MASK); +} + +static int hfc_upd_mix_info(struct rtw89_dev *rtwdev) +{ + const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def; + struct rtw89_hfc_param *param = &rtwdev->mac.hfc_param; + int ret; + + ret = rtw89_mac_check_mac_en(rtwdev, RTW89_MAC_0, RTW89_DMAC_SEL); + if (ret) + return ret; + + mac->hfc_get_mix_info(rtwdev); ret = hfc_pub_info_chk(rtwdev); if (param->en && ret) @@ -967,7 +1105,7 @@ static int hfc_upd_mix_info(struct rtw89_dev *rtwdev) return 0; } -static void hfc_h2c_cfg(struct rtw89_dev *rtwdev) +static void hfc_h2c_cfg_ax(struct rtw89_dev *rtwdev) { const struct rtw89_chip_info *chip = rtwdev->chip; const struct rtw89_page_regs *regs = chip->page_regs; @@ -983,7 +1121,7 @@ static void hfc_h2c_cfg(struct rtw89_dev *rtwdev) prec_cfg->h2c_full_cond); } -static void hfc_mix_cfg(struct rtw89_dev *rtwdev) +static void hfc_mix_cfg_ax(struct rtw89_dev *rtwdev) { const struct rtw89_chip_info *chip = rtwdev->chip; const struct rtw89_page_regs *regs = chip->page_regs; @@ -1018,7 +1156,7 @@ static void hfc_mix_cfg(struct rtw89_dev *rtwdev) rtw89_write32(rtwdev, regs->hci_fc_ctrl, val); } -static void hfc_func_en(struct rtw89_dev *rtwdev, bool en, bool h2c_en) +static void hfc_func_en_ax(struct rtw89_dev *rtwdev, bool en, bool h2c_en) { const struct rtw89_chip_info *chip = rtwdev->chip; const struct rtw89_page_regs *regs = chip->page_regs; @@ -1034,8 +1172,9 @@ static void hfc_func_en(struct rtw89_dev *rtwdev, bool en, bool h2c_en) rtw89_write32(rtwdev, regs->hci_fc_ctrl, val); } -static int hfc_init(struct rtw89_dev *rtwdev, bool reset, bool en, bool h2c_en) +int rtw89_mac_hfc_init(struct rtw89_dev *rtwdev, bool reset, bool en, bool h2c_en) { + const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def; const struct rtw89_chip_info *chip = rtwdev->chip; u32 dma_ch_mask = chip->dma_ch_mask; u8 ch; @@ -1050,11 +1189,11 @@ static int hfc_init(struct rtw89_dev *rtwdev, bool reset, bool en, bool h2c_en) if (ret) return ret; - hfc_func_en(rtwdev, false, false); + mac->hfc_func_en(rtwdev, false, false); if (!en && h2c_en) { - hfc_h2c_cfg(rtwdev); - hfc_func_en(rtwdev, en, h2c_en); + mac->hfc_h2c_cfg(rtwdev); + mac->hfc_func_en(rtwdev, en, h2c_en); return ret; } @@ -1070,9 +1209,9 @@ static int hfc_init(struct rtw89_dev *rtwdev, bool reset, bool en, bool h2c_en) if (ret) return ret; - hfc_mix_cfg(rtwdev); + mac->hfc_mix_cfg(rtwdev); if (en || h2c_en) { - hfc_func_en(rtwdev, en, h2c_en); + mac->hfc_func_en(rtwdev, en, h2c_en); udelay(10); } for (ch = RTW89_DMA_ACH0; ch < RTW89_DMA_H2C; ch++) { @@ -1334,9 +1473,14 @@ static int rtw89_mac_power_switch(struct rtw89_dev *rtwdev, bool on) if (on) { set_bit(RTW89_FLAG_POWERON, rtwdev->flags); + set_bit(RTW89_FLAG_DMAC_FUNC, rtwdev->flags); + set_bit(RTW89_FLAG_CMAC0_FUNC, rtwdev->flags); rtw89_write8(rtwdev, R_AX_SCOREBOARD + 3, MAC_AX_NOTIFY_TP_MAJOR); } else { clear_bit(RTW89_FLAG_POWERON, rtwdev->flags); + clear_bit(RTW89_FLAG_DMAC_FUNC, rtwdev->flags); + clear_bit(RTW89_FLAG_CMAC0_FUNC, rtwdev->flags); + clear_bit(RTW89_FLAG_CMAC1_FUNC, rtwdev->flags); clear_bit(RTW89_FLAG_FW_RDY, rtwdev->flags); rtw89_write8(rtwdev, R_AX_SCOREBOARD + 3, MAC_AX_NOTIFY_PWR_MAJOR); rtw89_set_entity_state(rtwdev, false); @@ -1351,7 +1495,7 @@ void rtw89_mac_pwr_off(struct rtw89_dev *rtwdev) rtw89_mac_power_switch(rtwdev, false); } -static int cmac_func_en(struct rtw89_dev *rtwdev, u8 mac_idx, bool en) +static int cmac_func_en_ax(struct rtw89_dev *rtwdev, u8 mac_idx, bool en) { u32 func_en = 0; u32 ck_en = 0; @@ -1397,7 +1541,7 @@ static int cmac_func_en(struct rtw89_dev *rtwdev, u8 mac_idx, bool en) return 0; } -static int dmac_func_en(struct rtw89_dev *rtwdev) +static int dmac_func_en_ax(struct rtw89_dev *rtwdev) { enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; u32 val32; @@ -1429,7 +1573,7 @@ static int dmac_func_en(struct rtw89_dev *rtwdev) return 0; } -static int chip_func_en(struct rtw89_dev *rtwdev) +static int chip_func_en_ax(struct rtw89_dev *rtwdev) { enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; @@ -1440,19 +1584,19 @@ static int chip_func_en(struct rtw89_dev *rtwdev) return 0; } -static int rtw89_mac_sys_init(struct rtw89_dev *rtwdev) +static int sys_init_ax(struct rtw89_dev *rtwdev) { int ret; - ret = dmac_func_en(rtwdev); + ret = dmac_func_en_ax(rtwdev); if (ret) return ret; - ret = cmac_func_en(rtwdev, 0, true); + ret = cmac_func_en_ax(rtwdev, 0, true); if (ret) return ret; - ret = chip_func_en(rtwdev); + ret = chip_func_en_ax(rtwdev); if (ret) return ret; @@ -1461,10 +1605,14 @@ static int rtw89_mac_sys_init(struct rtw89_dev *rtwdev) const struct rtw89_mac_size_set rtw89_mac_size = { .hfc_preccfg_pcie = {2, 40, 0, 0, 1, 0, 0, 0}, + .hfc_prec_cfg_c0 = {2, 32, 0, 0, 0, 0, 0, 0}, + .hfc_prec_cfg_c2 = {0, 256, 0, 0, 0, 0, 0, 0}, /* PCIE 64 */ .wde_size0 = {RTW89_WDE_PG_64, 4095, 1,}, + .wde_size0_v1 = {RTW89_WDE_PG_64, 3328, 0, 0,}, /* DLFW */ .wde_size4 = {RTW89_WDE_PG_64, 0, 4096,}, + .wde_size4_v1 = {RTW89_WDE_PG_64, 0, 3328, 0,}, /* PCIE 64 */ .wde_size6 = {RTW89_WDE_PG_64, 512, 0,}, /* 8852B PCIE SCC */ @@ -1477,6 +1625,8 @@ const struct rtw89_mac_size_set rtw89_mac_size = { .wde_size19 = {RTW89_WDE_PG_64, 3328, 0,}, /* PCIE */ .ple_size0 = {RTW89_PLE_PG_128, 1520, 16,}, + .ple_size0_v1 = {RTW89_PLE_PG_128, 2672, 256, 212992,}, + .ple_size3_v1 = {RTW89_PLE_PG_128, 2928, 0, 212992,}, /* DLFW */ .ple_size4 = {RTW89_PLE_PG_128, 64, 1472,}, /* PCIE 64 */ @@ -1489,6 +1639,7 @@ const struct rtw89_mac_size_set rtw89_mac_size = { .ple_size19 = {RTW89_PLE_PG_128, 1904, 16,}, /* PCIE 64 */ .wde_qt0 = {3792, 196, 0, 107,}, + .wde_qt0_v1 = {3302, 6, 0, 20,}, /* DLFW */ .wde_qt4 = {0, 0, 0, 0,}, /* PCIE 64 */ @@ -1499,10 +1650,13 @@ const struct rtw89_mac_size_set rtw89_mac_size = { .wde_qt17 = {0, 0, 0, 0,}, /* 8852C PCIE SCC */ .wde_qt18 = {3228, 60, 0, 40,}, + .ple_qt0 = {320, 0, 32, 16, 13, 13, 292, 0, 32, 18, 1, 4, 0,}, + .ple_qt1 = {320, 0, 32, 16, 1944, 1944, 2223, 0, 1963, 1949, 1, 1935, 0,}, /* PCIE SCC */ .ple_qt4 = {264, 0, 16, 20, 26, 13, 356, 0, 32, 40, 8,}, /* PCIE SCC */ .ple_qt5 = {264, 0, 32, 20, 64, 13, 1101, 0, 64, 128, 120,}, + .ple_qt9 = {0, 0, 32, 256, 0, 0, 0, 0, 0, 0, 1, 0, 0,}, /* DLFW */ .ple_qt13 = {0, 0, 16, 48, 0, 0, 0, 0, 0, 0, 0,}, /* PCIE 64 */ @@ -1523,6 +1677,10 @@ const struct rtw89_mac_size_set rtw89_mac_size = { .ple_qt_52b_wow = {147, 0, 16, 20, 157, 13, 133, 0, 172, 14, 24, 0,}, /* 8851B PCIE WOW */ .ple_qt_51b_wow = {147, 0, 16, 20, 157, 13, 133, 0, 172, 14, 24, 0,}, + .ple_rsvd_qt0 = {2, 112, 56, 6, 6, 6, 6, 0, 0, 62,}, + .ple_rsvd_qt1 = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0,}, + .rsvd0_size0 = {212992, 0,}, + .rsvd1_size0 = {587776, 2048,}, }; EXPORT_SYMBOL(rtw89_mac_size); @@ -1541,7 +1699,9 @@ static const struct rtw89_dle_mem *get_dle_mem_cfg(struct rtw89_dev *rtwdev, return NULL; } + mac->dle_info.rsvd_qt = cfg->rsvd_qt; mac->dle_info.ple_pg_size = cfg->ple_size->pge_size; + mac->dle_info.ple_free_pg = cfg->ple_size->lnk_pge_num; mac->dle_info.qta_mode = mode; mac->dle_info.c0_rx_qta = cfg->ple_min_qt->cma0_dma; mac->dle_info.c1_rx_qta = cfg->ple_min_qt->cma1_dma; @@ -1549,33 +1709,86 @@ static const struct rtw89_dle_mem *get_dle_mem_cfg(struct rtw89_dev *rtwdev, return cfg; } -static bool mac_is_txq_empty(struct rtw89_dev *rtwdev) +int rtw89_mac_get_dle_rsvd_qt_cfg(struct rtw89_dev *rtwdev, + enum rtw89_mac_dle_rsvd_qt_type type, + struct rtw89_mac_dle_rsvd_qt_cfg *cfg) +{ + struct rtw89_dle_info *dle_info = &rtwdev->mac.dle_info; + const struct rtw89_rsvd_quota *rsvd_qt = dle_info->rsvd_qt; + + switch (type) { + case DLE_RSVD_QT_MPDU_INFO: + cfg->pktid = dle_info->ple_free_pg; + cfg->pg_num = rsvd_qt->mpdu_info_tbl; + break; + case DLE_RSVD_QT_B0_CSI: + cfg->pktid = dle_info->ple_free_pg + rsvd_qt->mpdu_info_tbl; + cfg->pg_num = rsvd_qt->b0_csi; + break; + case DLE_RSVD_QT_B1_CSI: + cfg->pktid = dle_info->ple_free_pg + + rsvd_qt->mpdu_info_tbl + rsvd_qt->b0_csi; + cfg->pg_num = rsvd_qt->b1_csi; + break; + case DLE_RSVD_QT_B0_LMR: + cfg->pktid = dle_info->ple_free_pg + + rsvd_qt->mpdu_info_tbl + rsvd_qt->b0_csi + rsvd_qt->b1_csi; + cfg->pg_num = rsvd_qt->b0_lmr; + break; + case DLE_RSVD_QT_B1_LMR: + cfg->pktid = dle_info->ple_free_pg + + rsvd_qt->mpdu_info_tbl + rsvd_qt->b0_csi + rsvd_qt->b1_csi + + rsvd_qt->b0_lmr; + cfg->pg_num = rsvd_qt->b1_lmr; + break; + case DLE_RSVD_QT_B0_FTM: + cfg->pktid = dle_info->ple_free_pg + + rsvd_qt->mpdu_info_tbl + rsvd_qt->b0_csi + rsvd_qt->b1_csi + + rsvd_qt->b0_lmr + rsvd_qt->b1_lmr; + cfg->pg_num = rsvd_qt->b0_ftm; + break; + case DLE_RSVD_QT_B1_FTM: + cfg->pktid = dle_info->ple_free_pg + + rsvd_qt->mpdu_info_tbl + rsvd_qt->b0_csi + rsvd_qt->b1_csi + + rsvd_qt->b0_lmr + rsvd_qt->b1_lmr + rsvd_qt->b0_ftm; + cfg->pg_num = rsvd_qt->b1_ftm; + break; + default: + return -EINVAL; + } + + cfg->size = (u32)cfg->pg_num * dle_info->ple_pg_size; + + return 0; +} + +static bool mac_is_txq_empty_ax(struct rtw89_dev *rtwdev) { struct rtw89_mac_dle_dfi_qempty qempty; - u32 qnum, qtmp, val32, msk32; + u32 grpnum, qtmp, val32, msk32; int i, j, ret; - qnum = rtwdev->chip->wde_qempty_acq_num; + grpnum = rtwdev->chip->wde_qempty_acq_grpnum; qempty.dle_type = DLE_CTRL_TYPE_WDE; - for (i = 0; i < qnum; i++) { + for (i = 0; i < grpnum; i++) { qempty.grpsel = i; - ret = dle_dfi_qempty(rtwdev, &qempty); + ret = rtw89_mac_dle_dfi_qempty_cfg(rtwdev, &qempty); if (ret) { rtw89_warn(rtwdev, "dle dfi acq empty %d\n", ret); return false; } qtmp = qempty.qempty; for (j = 0 ; j < QEMP_ACQ_GRP_MACID_NUM; j++) { - val32 = FIELD_GET(QEMP_ACQ_GRP_QSEL_MASK, qtmp); + val32 = u32_get_bits(qtmp, QEMP_ACQ_GRP_QSEL_MASK); if (val32 != QEMP_ACQ_GRP_QSEL_MASK) return false; qtmp >>= QEMP_ACQ_GRP_QSEL_SH; } } - qempty.grpsel = rtwdev->chip->wde_qempty_mgq_sel; - ret = dle_dfi_qempty(rtwdev, &qempty); + qempty.grpsel = rtwdev->chip->wde_qempty_mgq_grpsel; + ret = rtw89_mac_dle_dfi_qempty_cfg(rtwdev, &qempty); if (ret) { rtw89_warn(rtwdev, "dle dfi mgq empty %d\n", ret); return false; @@ -1603,11 +1816,21 @@ static bool mac_is_txq_empty(struct rtw89_dev *rtwdev) return (val32 & msk32) == msk32; } -static inline u32 dle_used_size(const struct rtw89_dle_size *wde, - const struct rtw89_dle_size *ple) +static inline u32 dle_used_size(const struct rtw89_dle_mem *cfg) { - return wde->pge_size * (wde->lnk_pge_num + wde->unlnk_pge_num) + + const struct rtw89_dle_size *wde = cfg->wde_size; + const struct rtw89_dle_size *ple = cfg->ple_size; + u32 used; + + used = wde->pge_size * (wde->lnk_pge_num + wde->unlnk_pge_num) + ple->pge_size * (ple->lnk_pge_num + ple->unlnk_pge_num); + + if (cfg->rsvd0_size && cfg->rsvd1_size) { + used += cfg->rsvd0_size->size; + used += cfg->rsvd1_size->size; + } + + return used; } static u32 dle_expected_used_size(struct rtw89_dev *rtwdev, @@ -1621,7 +1844,7 @@ static u32 dle_expected_used_size(struct rtw89_dev *rtwdev, return size; } -static void dle_func_en(struct rtw89_dev *rtwdev, bool enable) +static void dle_func_en_ax(struct rtw89_dev *rtwdev, bool enable) { if (enable) rtw89_write32_set(rtwdev, R_AX_DMAC_FUNC_EN, @@ -1631,7 +1854,7 @@ static void dle_func_en(struct rtw89_dev *rtwdev, bool enable) B_AX_DLE_WDE_EN | B_AX_DLE_PLE_EN); } -static void dle_clk_en(struct rtw89_dev *rtwdev, bool enable) +static void dle_clk_en_ax(struct rtw89_dev *rtwdev, bool enable) { u32 val = B_AX_DLE_WDE_CLK_EN | B_AX_DLE_PLE_CLK_EN; @@ -1644,7 +1867,7 @@ static void dle_clk_en(struct rtw89_dev *rtwdev, bool enable) } } -static int dle_mix_cfg(struct rtw89_dev *rtwdev, const struct rtw89_dle_mem *cfg) +static int dle_mix_cfg_ax(struct rtw89_dev *rtwdev, const struct rtw89_dle_mem *cfg) { const struct rtw89_dle_size *size_cfg; u32 val; @@ -1701,6 +1924,23 @@ static int dle_mix_cfg(struct rtw89_dev *rtwdev, const struct rtw89_dle_mem *cfg return 0; } +static int chk_dle_rdy_ax(struct rtw89_dev *rtwdev, bool wde_or_ple) +{ + u32 reg, mask; + u32 ini; + + if (wde_or_ple) { + reg = R_AX_WDE_INI_STATUS; + mask = WDE_MGN_INI_RDY; + } else { + reg = R_AX_PLE_INI_STATUS; + mask = PLE_MGN_INI_RDY; + } + + return read_poll_timeout(rtw89_read32, ini, (ini & mask) == mask, 1, + 2000, false, rtwdev, reg); +} + #define INVALID_QT_WCPU U16_MAX #define SET_QUOTA_VAL(_min_x, _max_x, _module, _idx) \ do { \ @@ -1713,10 +1953,10 @@ static int dle_mix_cfg(struct rtw89_dev *rtwdev, const struct rtw89_dle_mem *cfg #define SET_QUOTA(_x, _module, _idx) \ SET_QUOTA_VAL(min_cfg->_x, max_cfg->_x, _module, _idx) -static void wde_quota_cfg(struct rtw89_dev *rtwdev, - const struct rtw89_wde_quota *min_cfg, - const struct rtw89_wde_quota *max_cfg, - u16 ext_wde_min_qt_wcpu) +static void wde_quota_cfg_ax(struct rtw89_dev *rtwdev, + const struct rtw89_wde_quota *min_cfg, + const struct rtw89_wde_quota *max_cfg, + u16 ext_wde_min_qt_wcpu) { u16 min_qt_wcpu = ext_wde_min_qt_wcpu != INVALID_QT_WCPU ? ext_wde_min_qt_wcpu : min_cfg->wcpu; @@ -1728,9 +1968,9 @@ static void wde_quota_cfg(struct rtw89_dev *rtwdev, SET_QUOTA(cpu_io, WDE, 4); } -static void ple_quota_cfg(struct rtw89_dev *rtwdev, - const struct rtw89_ple_quota *min_cfg, - const struct rtw89_ple_quota *max_cfg) +static void ple_quota_cfg_ax(struct rtw89_dev *rtwdev, + const struct rtw89_ple_quota *min_cfg, + const struct rtw89_ple_quota *max_cfg) { u32 val; @@ -1795,17 +2035,19 @@ static void dle_quota_cfg(struct rtw89_dev *rtwdev, const struct rtw89_dle_mem *cfg, u16 ext_wde_min_qt_wcpu) { - wde_quota_cfg(rtwdev, cfg->wde_min_qt, cfg->wde_max_qt, ext_wde_min_qt_wcpu); - ple_quota_cfg(rtwdev, cfg->ple_min_qt, cfg->ple_max_qt); + const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def; + + mac->wde_quota_cfg(rtwdev, cfg->wde_min_qt, cfg->wde_max_qt, ext_wde_min_qt_wcpu); + mac->ple_quota_cfg(rtwdev, cfg->ple_min_qt, cfg->ple_max_qt); } -static int dle_init(struct rtw89_dev *rtwdev, enum rtw89_qta_mode mode, - enum rtw89_qta_mode ext_mode) +int rtw89_mac_dle_init(struct rtw89_dev *rtwdev, enum rtw89_qta_mode mode, + enum rtw89_qta_mode ext_mode) { + const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def; const struct rtw89_dle_mem *cfg, *ext_cfg; u16 ext_wde_min_qt_wcpu = INVALID_QT_WCPU; - int ret = 0; - u32 ini; + int ret; ret = rtw89_mac_check_mac_en(rtwdev, RTW89_MAC_0, RTW89_DMAC_SEL); if (ret) @@ -1829,36 +2071,31 @@ static int dle_init(struct rtw89_dev *rtwdev, enum rtw89_qta_mode mode, ext_wde_min_qt_wcpu = ext_cfg->wde_min_qt->wcpu; } - if (dle_used_size(cfg->wde_size, cfg->ple_size) != - dle_expected_used_size(rtwdev, mode)) { + if (dle_used_size(cfg) != dle_expected_used_size(rtwdev, mode)) { rtw89_err(rtwdev, "[ERR]wd/dle mem cfg\n"); ret = -EINVAL; goto error; } - dle_func_en(rtwdev, false); - dle_clk_en(rtwdev, true); + mac->dle_func_en(rtwdev, false); + mac->dle_clk_en(rtwdev, true); - ret = dle_mix_cfg(rtwdev, cfg); + ret = mac->dle_mix_cfg(rtwdev, cfg); if (ret) { rtw89_err(rtwdev, "[ERR] dle mix cfg\n"); goto error; } dle_quota_cfg(rtwdev, cfg, ext_wde_min_qt_wcpu); - dle_func_en(rtwdev, true); + mac->dle_func_en(rtwdev, true); - ret = read_poll_timeout(rtw89_read32, ini, - (ini & WDE_MGN_INI_RDY) == WDE_MGN_INI_RDY, 1, - 2000, false, rtwdev, R_AX_WDE_INI_STATUS); + ret = mac->chk_dle_rdy(rtwdev, true); if (ret) { rtw89_err(rtwdev, "[ERR]WDE cfg ready\n"); return ret; } - ret = read_poll_timeout(rtw89_read32, ini, - (ini & WDE_MGN_INI_RDY) == WDE_MGN_INI_RDY, 1, - 2000, false, rtwdev, R_AX_PLE_INI_STATUS); + ret = mac->chk_dle_rdy(rtwdev, false); if (ret) { rtw89_err(rtwdev, "[ERR]PLE cfg ready\n"); return ret; @@ -1866,7 +2103,7 @@ static int dle_init(struct rtw89_dev *rtwdev, enum rtw89_qta_mode mode, return 0; error: - dle_func_en(rtwdev, false); + mac->dle_func_en(rtwdev, false); rtw89_err(rtwdev, "[ERR]trxcfg wde 0x8900 = %x\n", rtw89_read32(rtwdev, R_AX_WDE_INI_STATUS)); rtw89_err(rtwdev, "[ERR]trxcfg ple 0x8D00 = %x\n", @@ -1901,8 +2138,8 @@ static bool is_qta_poh(struct rtw89_dev *rtwdev) return rtwdev->hci.type == RTW89_HCI_TYPE_PCIE; } -static int preload_init(struct rtw89_dev *rtwdev, enum rtw89_mac_idx mac_idx, - enum rtw89_qta_mode mode) +int rtw89_mac_preload_init(struct rtw89_dev *rtwdev, enum rtw89_mac_idx mac_idx, + enum rtw89_qta_mode mode) { const struct rtw89_chip_info *chip = rtwdev->chip; @@ -1951,7 +2188,7 @@ static void _patch_ss2f_path(struct rtw89_dev *rtwdev) SS2F_PATH_WLCPU); } -static int sta_sch_init(struct rtw89_dev *rtwdev) +static int sta_sch_init_ax(struct rtw89_dev *rtwdev) { u32 p_val; u8 val; @@ -1980,7 +2217,7 @@ static int sta_sch_init(struct rtw89_dev *rtwdev) return 0; } -static int mpdu_proc_init(struct rtw89_dev *rtwdev) +static int mpdu_proc_init_ax(struct rtw89_dev *rtwdev) { int ret; @@ -1997,7 +2234,7 @@ static int mpdu_proc_init(struct rtw89_dev *rtwdev) return 0; } -static int sec_eng_init(struct rtw89_dev *rtwdev) +static int sec_eng_init_ax(struct rtw89_dev *rtwdev) { const struct rtw89_chip_info *chip = rtwdev->chip; u32 val = 0; @@ -2032,41 +2269,41 @@ static int sec_eng_init(struct rtw89_dev *rtwdev) return 0; } -static int dmac_init(struct rtw89_dev *rtwdev, u8 mac_idx) +static int dmac_init_ax(struct rtw89_dev *rtwdev, u8 mac_idx) { int ret; - ret = dle_init(rtwdev, rtwdev->mac.qta_mode, RTW89_QTA_INVALID); + ret = rtw89_mac_dle_init(rtwdev, rtwdev->mac.qta_mode, RTW89_QTA_INVALID); if (ret) { rtw89_err(rtwdev, "[ERR]DLE init %d\n", ret); return ret; } - ret = preload_init(rtwdev, RTW89_MAC_0, rtwdev->mac.qta_mode); + ret = rtw89_mac_preload_init(rtwdev, RTW89_MAC_0, rtwdev->mac.qta_mode); if (ret) { rtw89_err(rtwdev, "[ERR]preload init %d\n", ret); return ret; } - ret = hfc_init(rtwdev, true, true, true); + ret = rtw89_mac_hfc_init(rtwdev, true, true, true); if (ret) { rtw89_err(rtwdev, "[ERR]HCI FC init %d\n", ret); return ret; } - ret = sta_sch_init(rtwdev); + ret = sta_sch_init_ax(rtwdev); if (ret) { rtw89_err(rtwdev, "[ERR]STA SCH init %d\n", ret); return ret; } - ret = mpdu_proc_init(rtwdev); + ret = mpdu_proc_init_ax(rtwdev); if (ret) { rtw89_err(rtwdev, "[ERR]MPDU Proc init %d\n", ret); return ret; } - ret = sec_eng_init(rtwdev); + ret = sec_eng_init_ax(rtwdev); if (ret) { rtw89_err(rtwdev, "[ERR]Security Engine init %d\n", ret); return ret; @@ -2075,7 +2312,7 @@ static int dmac_init(struct rtw89_dev *rtwdev, u8 mac_idx) return ret; } -static int addr_cam_init(struct rtw89_dev *rtwdev, u8 mac_idx) +static int addr_cam_init_ax(struct rtw89_dev *rtwdev, u8 mac_idx) { u32 val, reg; u16 p_val; @@ -2102,7 +2339,7 @@ static int addr_cam_init(struct rtw89_dev *rtwdev, u8 mac_idx) return 0; } -static int scheduler_init(struct rtw89_dev *rtwdev, u8 mac_idx) +static int scheduler_init_ax(struct rtw89_dev *rtwdev, u8 mac_idx) { u32 ret; u32 reg; @@ -2143,10 +2380,10 @@ static int scheduler_init(struct rtw89_dev *rtwdev, u8 mac_idx) return 0; } -int rtw89_mac_typ_fltr_opt(struct rtw89_dev *rtwdev, - enum rtw89_machdr_frame_type type, - enum rtw89_mac_fwd_target fwd_target, - u8 mac_idx) +static int rtw89_mac_typ_fltr_opt_ax(struct rtw89_dev *rtwdev, + enum rtw89_machdr_frame_type type, + enum rtw89_mac_fwd_target fwd_target, + u8 mac_idx) { u32 reg; u32 val; @@ -2185,7 +2422,7 @@ int rtw89_mac_typ_fltr_opt(struct rtw89_dev *rtwdev, return 0; } -static int rx_fltr_init(struct rtw89_dev *rtwdev, u8 mac_idx) +static int rx_fltr_init_ax(struct rtw89_dev *rtwdev, u8 mac_idx) { int ret, i; u32 mac_ftlr, plcp_ftlr; @@ -2195,8 +2432,8 @@ static int rx_fltr_init(struct rtw89_dev *rtwdev, u8 mac_idx) return ret; for (i = RTW89_MGNT; i <= RTW89_DATA; i++) { - ret = rtw89_mac_typ_fltr_opt(rtwdev, i, RTW89_FWD_TO_HOST, - mac_idx); + ret = rtw89_mac_typ_fltr_opt_ax(rtwdev, i, RTW89_FWD_TO_HOST, + mac_idx); if (ret) return ret; } @@ -2247,7 +2484,7 @@ static void _patch_dis_resp_chk(struct rtw89_dev *rtwdev, u8 mac_idx) } } -static int cca_ctrl_init(struct rtw89_dev *rtwdev, u8 mac_idx) +static int cca_ctrl_init_ax(struct rtw89_dev *rtwdev, u8 mac_idx) { u32 val, reg; int ret; @@ -2279,7 +2516,7 @@ static int cca_ctrl_init(struct rtw89_dev *rtwdev, u8 mac_idx) return 0; } -static int nav_ctrl_init(struct rtw89_dev *rtwdev) +static int nav_ctrl_init_ax(struct rtw89_dev *rtwdev) { rtw89_write32_set(rtwdev, R_AX_WMAC_NAV_CTL, B_AX_WMAC_PLCP_UP_NAV_EN | B_AX_WMAC_TF_UP_NAV_EN | @@ -2289,7 +2526,7 @@ static int nav_ctrl_init(struct rtw89_dev *rtwdev) return 0; } -static int spatial_reuse_init(struct rtw89_dev *rtwdev, u8 mac_idx) +static int spatial_reuse_init_ax(struct rtw89_dev *rtwdev, u8 mac_idx) { u32 reg; int ret; @@ -2303,7 +2540,7 @@ static int spatial_reuse_init(struct rtw89_dev *rtwdev, u8 mac_idx) return 0; } -static int tmac_init(struct rtw89_dev *rtwdev, u8 mac_idx) +static int tmac_init_ax(struct rtw89_dev *rtwdev, u8 mac_idx) { u32 reg; int ret; @@ -2325,7 +2562,7 @@ static int tmac_init(struct rtw89_dev *rtwdev, u8 mac_idx) return 0; } -static int trxptcl_init(struct rtw89_dev *rtwdev, u8 mac_idx) +static int trxptcl_init_ax(struct rtw89_dev *rtwdev, u8 mac_idx) { const struct rtw89_chip_info *chip = rtwdev->chip; const struct rtw89_rrsr_cfgs *rrsr = chip->rrsr_cfgs; @@ -2382,7 +2619,7 @@ static void rst_bacam(struct rtw89_dev *rtwdev) rtw89_warn(rtwdev, "failed to reset BA CAM\n"); } -static int rmac_init(struct rtw89_dev *rtwdev, u8 mac_idx) +static int rmac_init_ax(struct rtw89_dev *rtwdev, u8 mac_idx) { #define TRXCFG_RMAC_CCA_TO 32 #define TRXCFG_RMAC_DATA_TO 15 @@ -2440,7 +2677,7 @@ static int rmac_init(struct rtw89_dev *rtwdev, u8 mac_idx) return ret; } -static int cmac_com_init(struct rtw89_dev *rtwdev, u8 mac_idx) +static int cmac_com_init_ax(struct rtw89_dev *rtwdev, u8 mac_idx) { enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; u32 val, reg; @@ -2465,7 +2702,7 @@ static int cmac_com_init(struct rtw89_dev *rtwdev, u8 mac_idx) return 0; } -static bool is_qta_dbcc(struct rtw89_dev *rtwdev, enum rtw89_qta_mode mode) +bool rtw89_mac_is_qta_dbcc(struct rtw89_dev *rtwdev, enum rtw89_qta_mode mode) { const struct rtw89_dle_mem *cfg; @@ -2478,7 +2715,7 @@ static bool is_qta_dbcc(struct rtw89_dev *rtwdev, enum rtw89_qta_mode mode) return (cfg->ple_min_qt->cma1_dma && cfg->ple_max_qt->cma1_dma); } -static int ptcl_init(struct rtw89_dev *rtwdev, u8 mac_idx) +static int ptcl_init_ax(struct rtw89_dev *rtwdev, u8 mac_idx) { u32 val, reg; int ret; @@ -2521,7 +2758,7 @@ static int ptcl_init(struct rtw89_dev *rtwdev, u8 mac_idx) return 0; } -static int cmac_dma_init(struct rtw89_dev *rtwdev, u8 mac_idx) +static int cmac_dma_init_ax(struct rtw89_dev *rtwdev, u8 mac_idx) { enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; u32 reg; @@ -2540,82 +2777,82 @@ static int cmac_dma_init(struct rtw89_dev *rtwdev, u8 mac_idx) return 0; } -static int cmac_init(struct rtw89_dev *rtwdev, u8 mac_idx) +static int cmac_init_ax(struct rtw89_dev *rtwdev, u8 mac_idx) { int ret; - ret = scheduler_init(rtwdev, mac_idx); + ret = scheduler_init_ax(rtwdev, mac_idx); if (ret) { rtw89_err(rtwdev, "[ERR]CMAC%d SCH init %d\n", mac_idx, ret); return ret; } - ret = addr_cam_init(rtwdev, mac_idx); + ret = addr_cam_init_ax(rtwdev, mac_idx); if (ret) { rtw89_err(rtwdev, "[ERR]CMAC%d ADDR_CAM reset %d\n", mac_idx, ret); return ret; } - ret = rx_fltr_init(rtwdev, mac_idx); + ret = rx_fltr_init_ax(rtwdev, mac_idx); if (ret) { rtw89_err(rtwdev, "[ERR]CMAC%d RX filter init %d\n", mac_idx, ret); return ret; } - ret = cca_ctrl_init(rtwdev, mac_idx); + ret = cca_ctrl_init_ax(rtwdev, mac_idx); if (ret) { rtw89_err(rtwdev, "[ERR]CMAC%d CCA CTRL init %d\n", mac_idx, ret); return ret; } - ret = nav_ctrl_init(rtwdev); + ret = nav_ctrl_init_ax(rtwdev); if (ret) { rtw89_err(rtwdev, "[ERR]CMAC%d NAV CTRL init %d\n", mac_idx, ret); return ret; } - ret = spatial_reuse_init(rtwdev, mac_idx); + ret = spatial_reuse_init_ax(rtwdev, mac_idx); if (ret) { rtw89_err(rtwdev, "[ERR]CMAC%d Spatial Reuse init %d\n", mac_idx, ret); return ret; } - ret = tmac_init(rtwdev, mac_idx); + ret = tmac_init_ax(rtwdev, mac_idx); if (ret) { rtw89_err(rtwdev, "[ERR]CMAC%d TMAC init %d\n", mac_idx, ret); return ret; } - ret = trxptcl_init(rtwdev, mac_idx); + ret = trxptcl_init_ax(rtwdev, mac_idx); if (ret) { rtw89_err(rtwdev, "[ERR]CMAC%d TRXPTCL init %d\n", mac_idx, ret); return ret; } - ret = rmac_init(rtwdev, mac_idx); + ret = rmac_init_ax(rtwdev, mac_idx); if (ret) { rtw89_err(rtwdev, "[ERR]CMAC%d RMAC init %d\n", mac_idx, ret); return ret; } - ret = cmac_com_init(rtwdev, mac_idx); + ret = cmac_com_init_ax(rtwdev, mac_idx); if (ret) { rtw89_err(rtwdev, "[ERR]CMAC%d Com init %d\n", mac_idx, ret); return ret; } - ret = ptcl_init(rtwdev, mac_idx); + ret = ptcl_init_ax(rtwdev, mac_idx); if (ret) { rtw89_err(rtwdev, "[ERR]CMAC%d PTCL init %d\n", mac_idx, ret); return ret; } - ret = cmac_dma_init(rtwdev, mac_idx); + ret = cmac_dma_init_ax(rtwdev, mac_idx); if (ret) { rtw89_err(rtwdev, "[ERR]CMAC%d DMA init %d\n", mac_idx, ret); return ret; @@ -2878,7 +3115,7 @@ int rtw89_mac_resume_sch_tx_v1(struct rtw89_dev *rtwdev, u8 mac_idx, u32 tx_en) } EXPORT_SYMBOL(rtw89_mac_resume_sch_tx_v1); -int rtw89_mac_dle_buf_req(struct rtw89_dev *rtwdev, u16 buf_len, bool wd, u16 *pkt_id) +static int dle_buf_req_ax(struct rtw89_dev *rtwdev, u16 buf_len, bool wd, u16 *pkt_id) { u32 val, reg; int ret; @@ -2902,7 +3139,7 @@ int rtw89_mac_dle_buf_req(struct rtw89_dev *rtwdev, u16 buf_len, bool wd, u16 *p return 0; } -int rtw89_mac_set_cpuio(struct rtw89_dev *rtwdev, +static int set_cpuio_ax(struct rtw89_dev *rtwdev, struct rtw89_cpuio_ctrl *ctrl_para, bool wd) { u32 val, cmd_type, reg; @@ -2955,8 +3192,9 @@ int rtw89_mac_set_cpuio(struct rtw89_dev *rtwdev, return 0; } -static int dle_quota_change(struct rtw89_dev *rtwdev, enum rtw89_qta_mode mode) +int rtw89_mac_dle_quota_change(struct rtw89_dev *rtwdev, enum rtw89_qta_mode mode) { + const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def; const struct rtw89_dle_mem *cfg; struct rtw89_cpuio_ctrl ctrl_para = {0}; u16 pkt_id; @@ -2968,15 +3206,14 @@ static int dle_quota_change(struct rtw89_dev *rtwdev, enum rtw89_qta_mode mode) return -EINVAL; } - if (dle_used_size(cfg->wde_size, cfg->ple_size) != - dle_expected_used_size(rtwdev, mode)) { + if (dle_used_size(cfg) != dle_expected_used_size(rtwdev, mode)) { rtw89_err(rtwdev, "[ERR]wd/dle mem cfg\n"); return -EINVAL; } dle_quota_cfg(rtwdev, cfg, INVALID_QT_WCPU); - ret = rtw89_mac_dle_buf_req(rtwdev, 0x20, true, &pkt_id); + ret = mac->dle_buf_req(rtwdev, 0x20, true, &pkt_id); if (ret) { rtw89_err(rtwdev, "[ERR]WDE DLE buf req\n"); return ret; @@ -2988,13 +3225,13 @@ static int dle_quota_change(struct rtw89_dev *rtwdev, enum rtw89_qta_mode mode) ctrl_para.pkt_num = 0; ctrl_para.dst_pid = WDE_DLE_PORT_ID_WDRLS; ctrl_para.dst_qid = WDE_DLE_QUEID_NO_REPORT; - ret = rtw89_mac_set_cpuio(rtwdev, &ctrl_para, true); + ret = mac->set_cpuio(rtwdev, &ctrl_para, true); if (ret) { rtw89_err(rtwdev, "[ERR]WDE DLE enqueue to head\n"); return -EFAULT; } - ret = rtw89_mac_dle_buf_req(rtwdev, 0x20, false, &pkt_id); + ret = mac->dle_buf_req(rtwdev, 0x20, false, &pkt_id); if (ret) { rtw89_err(rtwdev, "[ERR]PLE DLE buf req\n"); return ret; @@ -3006,7 +3243,7 @@ static int dle_quota_change(struct rtw89_dev *rtwdev, enum rtw89_qta_mode mode) ctrl_para.pkt_num = 0; ctrl_para.dst_pid = PLE_DLE_PORT_ID_PLRLS; ctrl_para.dst_qid = PLE_DLE_QUEID_NO_REPORT; - ret = rtw89_mac_set_cpuio(rtwdev, &ctrl_para, false); + ret = mac->set_cpuio(rtwdev, &ctrl_para, false); if (ret) { rtw89_err(rtwdev, "[ERR]PLE DLE enqueue to head\n"); return -EFAULT; @@ -3038,7 +3275,7 @@ static int band_idle_ck_b(struct rtw89_dev *rtwdev, u8 mac_idx) return 0; } -static int band1_enable(struct rtw89_dev *rtwdev) +static int band1_enable_ax(struct rtw89_dev *rtwdev) { int ret, i; u32 sleep_bak[4] = {0}; @@ -3064,7 +3301,7 @@ static int band1_enable(struct rtw89_dev *rtwdev) return ret; } - ret = dle_quota_change(rtwdev, rtwdev->mac.qta_mode); + ret = rtw89_mac_dle_quota_change(rtwdev, rtwdev->mac.qta_mode); if (ret) { rtw89_err(rtwdev, "[ERR]DLE quota change %d\n", ret); return ret; @@ -3081,13 +3318,13 @@ static int band1_enable(struct rtw89_dev *rtwdev) return ret; } - ret = cmac_func_en(rtwdev, 1, true); + ret = cmac_func_en_ax(rtwdev, 1, true); if (ret) { rtw89_err(rtwdev, "[ERR]CMAC1 func en %d\n", ret); return ret; } - ret = cmac_init(rtwdev, 1); + ret = cmac_init_ax(rtwdev, 1); if (ret) { rtw89_err(rtwdev, "[ERR]CMAC1 init %d\n", ret); return ret; @@ -3296,8 +3533,8 @@ static void rtw89_tmac_imr_enable(struct rtw89_dev *rtwdev, u8 mac_idx) rtw89_write32_set(rtwdev, reg, imr->tmac_imr_set); } -static int rtw89_mac_enable_imr(struct rtw89_dev *rtwdev, u8 mac_idx, - enum rtw89_mac_hwmod_sel sel) +static int enable_imr_ax(struct rtw89_dev *rtwdev, u8 mac_idx, + enum rtw89_mac_hwmod_sel sel) { int ret; @@ -3334,7 +3571,7 @@ static int rtw89_mac_enable_imr(struct rtw89_dev *rtwdev, u8 mac_idx, return 0; } -static void rtw89_mac_err_imr_ctrl(struct rtw89_dev *rtwdev, bool en) +static void err_imr_ctrl_ax(struct rtw89_dev *rtwdev, bool en) { enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; @@ -3347,18 +3584,18 @@ static void rtw89_mac_err_imr_ctrl(struct rtw89_dev *rtwdev, bool en) en ? CMAC1_ERR_IMR_EN : CMAC1_ERR_IMR_DIS); } -static int rtw89_mac_dbcc_enable(struct rtw89_dev *rtwdev, bool enable) +static int dbcc_enable_ax(struct rtw89_dev *rtwdev, bool enable) { int ret = 0; if (enable) { - ret = band1_enable(rtwdev); + ret = band1_enable_ax(rtwdev); if (ret) { rtw89_err(rtwdev, "[ERR] band1_enable %d\n", ret); return ret; } - ret = rtw89_mac_enable_imr(rtwdev, RTW89_MAC_1, RTW89_CMAC_SEL); + ret = enable_imr_ax(rtwdev, RTW89_MAC_1, RTW89_CMAC_SEL); if (ret) { rtw89_err(rtwdev, "[ERR] enable CMAC1 IMR %d\n", ret); return ret; @@ -3371,7 +3608,7 @@ static int rtw89_mac_dbcc_enable(struct rtw89_dev *rtwdev, bool enable) return 0; } -static int set_host_rpr(struct rtw89_dev *rtwdev) +static int set_host_rpr_ax(struct rtw89_dev *rtwdev) { if (rtwdev->hci.type == RTW89_HCI_TYPE_PCIE) { rtw89_write32_mask(rtwdev, R_AX_WDRLS_CFG, @@ -3391,46 +3628,46 @@ static int set_host_rpr(struct rtw89_dev *rtwdev) return 0; } -static int rtw89_mac_trx_init(struct rtw89_dev *rtwdev) +static int trx_init_ax(struct rtw89_dev *rtwdev) { enum rtw89_qta_mode qta_mode = rtwdev->mac.qta_mode; int ret; - ret = dmac_init(rtwdev, 0); + ret = dmac_init_ax(rtwdev, 0); if (ret) { rtw89_err(rtwdev, "[ERR]DMAC init %d\n", ret); return ret; } - ret = cmac_init(rtwdev, 0); + ret = cmac_init_ax(rtwdev, 0); if (ret) { rtw89_err(rtwdev, "[ERR]CMAC%d init %d\n", 0, ret); return ret; } - if (is_qta_dbcc(rtwdev, qta_mode)) { - ret = rtw89_mac_dbcc_enable(rtwdev, true); + if (rtw89_mac_is_qta_dbcc(rtwdev, qta_mode)) { + ret = dbcc_enable_ax(rtwdev, true); if (ret) { rtw89_err(rtwdev, "[ERR]dbcc_enable init %d\n", ret); return ret; } } - ret = rtw89_mac_enable_imr(rtwdev, RTW89_MAC_0, RTW89_DMAC_SEL); + ret = enable_imr_ax(rtwdev, RTW89_MAC_0, RTW89_DMAC_SEL); if (ret) { rtw89_err(rtwdev, "[ERR] enable DMAC IMR %d\n", ret); return ret; } - ret = rtw89_mac_enable_imr(rtwdev, RTW89_MAC_0, RTW89_CMAC_SEL); + ret = enable_imr_ax(rtwdev, RTW89_MAC_0, RTW89_CMAC_SEL); if (ret) { rtw89_err(rtwdev, "[ERR] to enable CMAC0 IMR %d\n", ret); return ret; } - rtw89_mac_err_imr_ctrl(rtwdev, true); + err_imr_ctrl_ax(rtwdev, true); - ret = set_host_rpr(rtwdev); + ret = set_host_rpr_ax(rtwdev); if (ret) { rtw89_err(rtwdev, "[ERR] set host rpr %d\n", ret); return ret; @@ -3521,11 +3758,10 @@ static int rtw89_mac_enable_cpu_ax(struct rtw89_dev *rtwdev, u8 boot_reason, return 0; } -static int rtw89_mac_dmac_pre_init(struct rtw89_dev *rtwdev) +static void rtw89_mac_hci_func_en_ax(struct rtw89_dev *rtwdev) { enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; u32 val; - int ret; if (chip_id == RTL8852C) val = B_AX_MAC_FUNC_EN | B_AX_DMAC_FUNC_EN | B_AX_DISPATCHER_EN | @@ -3534,6 +3770,12 @@ static int rtw89_mac_dmac_pre_init(struct rtw89_dev *rtwdev) val = B_AX_MAC_FUNC_EN | B_AX_DMAC_FUNC_EN | B_AX_DISPATCHER_EN | B_AX_PKT_BUF_EN; rtw89_write32(rtwdev, R_AX_DMAC_FUNC_EN, val); +} + +static void rtw89_mac_dmac_func_pre_en_ax(struct rtw89_dev *rtwdev) +{ + enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; + u32 val; if (chip_id == RTL8851B) val = B_AX_DISPATCHER_CLK_EN | B_AX_AXIDMA_CLK_EN; @@ -3542,7 +3784,7 @@ static int rtw89_mac_dmac_pre_init(struct rtw89_dev *rtwdev) rtw89_write32(rtwdev, R_AX_DMAC_CLK_EN, val); if (chip_id != RTL8852C) - goto dle; + return; val = rtw89_read32(rtwdev, R_AX_HAXI_INIT_CFG1); val &= ~(B_AX_DMA_MODE_MASK | B_AX_STOP_AXI_MST); @@ -3557,15 +3799,23 @@ static int rtw89_mac_dmac_pre_init(struct rtw89_dev *rtwdev) B_AX_STOP_CH12 | B_AX_STOP_ACH2); rtw89_write32_clr(rtwdev, R_AX_HAXI_DMA_STOP2, B_AX_STOP_CH10 | B_AX_STOP_CH11); rtw89_write32_set(rtwdev, R_AX_PLATFORM_ENABLE, B_AX_AXIDMA_EN); +} + +static int rtw89_mac_dmac_pre_init(struct rtw89_dev *rtwdev) +{ + const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def; + int ret; -dle: - ret = dle_init(rtwdev, RTW89_QTA_DLFW, rtwdev->mac.qta_mode); + mac->hci_func_en(rtwdev); + mac->dmac_func_pre_en(rtwdev); + + ret = rtw89_mac_dle_init(rtwdev, RTW89_QTA_DLFW, rtwdev->mac.qta_mode); if (ret) { rtw89_err(rtwdev, "[ERR]DLE pre init %d\n", ret); return ret; } - ret = hfc_init(rtwdev, true, false, true); + ret = rtw89_mac_hfc_init(rtwdev, true, false, true); if (ret) { rtw89_err(rtwdev, "[ERR]HCI FC pre init %d\n", ret); return ret; @@ -3639,6 +3889,7 @@ int rtw89_mac_partial_init(struct rtw89_dev *rtwdev, bool include_bb) int rtw89_mac_init(struct rtw89_dev *rtwdev) { + const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def; const struct rtw89_chip_info *chip = rtwdev->chip; bool include_bb = !!chip->bbmcu_nr; int ret; @@ -3651,11 +3902,11 @@ int rtw89_mac_init(struct rtw89_dev *rtwdev) if (ret) goto fail; - ret = rtw89_mac_sys_init(rtwdev); + ret = mac->sys_init(rtwdev); if (ret) goto fail; - ret = rtw89_mac_trx_init(rtwdev); + ret = mac->trx_init(rtwdev); if (ret) goto fail; @@ -3754,6 +4005,50 @@ static const struct rtw89_port_reg rtw89_port_base_ax = { R_AX_PORT_HGQ_WINDOW_CFG + 3}, }; +static void rtw89_mac_check_packet_ctrl(struct rtw89_dev *rtwdev, + struct rtw89_vif *rtwvif, u8 type) +{ + u8 mask = B_AX_PTCL_DBG_INFO_MASK_BY_PORT(rtwvif->port); + u32 reg_info, reg_ctrl; + u32 val; + int ret; + + reg_info = rtw89_mac_reg_by_idx(rtwdev, R_AX_PTCL_DBG_INFO, rtwvif->mac_idx); + reg_ctrl = rtw89_mac_reg_by_idx(rtwdev, R_AX_PTCL_DBG, rtwvif->mac_idx); + + rtw89_write32_mask(rtwdev, reg_ctrl, B_AX_PTCL_DBG_SEL_MASK, type); + rtw89_write32_set(rtwdev, reg_ctrl, B_AX_PTCL_DBG_EN); + fsleep(100); + + ret = read_poll_timeout(rtw89_read32_mask, val, val == 0, 1000, 100000, + true, rtwdev, reg_info, mask); + if (ret) + rtw89_warn(rtwdev, "Polling beacon packet empty fail\n"); +} + +static void rtw89_mac_bcn_drop(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif) +{ + const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def; + const struct rtw89_port_reg *p = mac->port_base; + + rtw89_write32_set(rtwdev, R_AX_BCN_DROP_ALL0, BIT(rtwvif->port)); + rtw89_write32_port_mask(rtwdev, rtwvif, p->tbtt_prohib, B_AX_TBTT_SETUP_MASK, 1); + rtw89_write32_port_mask(rtwdev, rtwvif, p->bcn_area, B_AX_BCN_MSK_AREA_MASK, 0); + rtw89_write32_port_mask(rtwdev, rtwvif, p->tbtt_prohib, B_AX_TBTT_HOLD_MASK, 0); + rtw89_write32_port_mask(rtwdev, rtwvif, p->bcn_early, B_AX_BCNERLY_MASK, 2); + rtw89_write16_port_mask(rtwdev, rtwvif, p->tbtt_early, B_AX_TBTTERLY_MASK, 1); + rtw89_write32_port_mask(rtwdev, rtwvif, p->bcn_space, B_AX_BCN_SPACE_MASK, 1); + rtw89_write32_port_set(rtwdev, rtwvif, p->port_cfg, B_AX_BCNTX_EN); + + rtw89_mac_check_packet_ctrl(rtwdev, rtwvif, AX_PTCL_DBG_BCNQ_NUM0); + if (rtwvif->port == RTW89_PORT_0) + rtw89_mac_check_packet_ctrl(rtwdev, rtwvif, AX_PTCL_DBG_BCNQ_NUM1); + + rtw89_write32_clr(rtwdev, R_AX_BCN_DROP_ALL0, BIT(rtwvif->port)); + rtw89_write32_port_clr(rtwdev, rtwvif, p->port_cfg, B_AX_TBTT_PROHIB_EN); + fsleep(2); +} + #define BCN_INTERVAL 100 #define BCN_ERLY_DEF 160 #define BCN_SETUP_DEF 2 @@ -3769,21 +4064,36 @@ static void rtw89_mac_port_cfg_func_sw(struct rtw89_dev *rtwdev, const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def; const struct rtw89_port_reg *p = mac->port_base; struct ieee80211_vif *vif = rtwvif_to_vif(rtwvif); + const struct rtw89_chip_info *chip = rtwdev->chip; + bool need_backup = false; + u32 backup_val; if (!rtw89_read32_port_mask(rtwdev, rtwvif, p->port_cfg, B_AX_PORT_FUNC_EN)) return; - rtw89_write32_port_clr(rtwdev, rtwvif, p->tbtt_prohib, B_AX_TBTT_SETUP_MASK); - rtw89_write32_port_mask(rtwdev, rtwvif, p->tbtt_prohib, B_AX_TBTT_HOLD_MASK, 1); - rtw89_write16_port_clr(rtwdev, rtwvif, p->tbtt_early, B_AX_TBTTERLY_MASK); - rtw89_write16_port_clr(rtwdev, rtwvif, p->bcn_early, B_AX_BCNERLY_MASK); + if (chip->chip_id == RTL8852A && rtwvif->port != RTW89_PORT_0) { + need_backup = true; + backup_val = rtw89_read32_port(rtwdev, rtwvif, p->tbtt_prohib); + } + + if (rtwvif->net_type == RTW89_NET_TYPE_AP_MODE) + rtw89_mac_bcn_drop(rtwdev, rtwvif); - msleep(vif->bss_conf.beacon_int + 1); + if (chip->chip_id == RTL8852A) { + rtw89_write32_port_clr(rtwdev, rtwvif, p->tbtt_prohib, B_AX_TBTT_SETUP_MASK); + rtw89_write32_port_mask(rtwdev, rtwvif, p->tbtt_prohib, B_AX_TBTT_HOLD_MASK, 1); + rtw89_write16_port_clr(rtwdev, rtwvif, p->tbtt_early, B_AX_TBTTERLY_MASK); + rtw89_write16_port_clr(rtwdev, rtwvif, p->bcn_early, B_AX_BCNERLY_MASK); + } + msleep(vif->bss_conf.beacon_int + 1); rtw89_write32_port_clr(rtwdev, rtwvif, p->port_cfg, B_AX_PORT_FUNC_EN | B_AX_BRK_SETUP); rtw89_write32_port_set(rtwdev, rtwvif, p->port_cfg, B_AX_TSFTR_RST); rtw89_write32_port(rtwdev, rtwvif, p->bcn_cnt_tmr, 0); + + if (need_backup) + rtw89_write32_port(rtwdev, rtwvif, p->tbtt_prohib, backup_val); } static void rtw89_mac_port_cfg_tx_rpt(struct rtw89_dev *rtwdev, @@ -3864,12 +4174,10 @@ static void rtw89_mac_port_cfg_rx_sync(struct rtw89_dev *rtwdev, } static void rtw89_mac_port_cfg_tx_sw(struct rtw89_dev *rtwdev, - struct rtw89_vif *rtwvif) + struct rtw89_vif *rtwvif, bool en) { const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def; const struct rtw89_port_reg *p = mac->port_base; - bool en = rtwvif->net_type == RTW89_NET_TYPE_AP_MODE || - rtwvif->net_type == RTW89_NET_TYPE_AD_HOC; if (en) rtw89_write32_port_set(rtwdev, rtwvif, p->port_cfg, B_AX_BCNTX_EN); @@ -3877,6 +4185,24 @@ static void rtw89_mac_port_cfg_tx_sw(struct rtw89_dev *rtwdev, rtw89_write32_port_clr(rtwdev, rtwvif, p->port_cfg, B_AX_BCNTX_EN); } +static void rtw89_mac_port_cfg_tx_sw_by_nettype(struct rtw89_dev *rtwdev, + struct rtw89_vif *rtwvif) +{ + bool en = rtwvif->net_type == RTW89_NET_TYPE_AP_MODE || + rtwvif->net_type == RTW89_NET_TYPE_AD_HOC; + + rtw89_mac_port_cfg_tx_sw(rtwdev, rtwvif, en); +} + +void rtw89_mac_enable_beacon_for_ap_vifs(struct rtw89_dev *rtwdev, bool en) +{ + struct rtw89_vif *rtwvif; + + rtw89_for_each_rtwvif(rtwdev, rtwvif) + if (rtwvif->net_type == RTW89_NET_TYPE_AP_MODE) + rtw89_mac_port_cfg_tx_sw(rtwdev, rtwvif, en); +} + static void rtw89_mac_port_cfg_bcn_intv(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif) { @@ -4183,7 +4509,7 @@ int rtw89_mac_port_update(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif) rtw89_mac_port_cfg_bcn_prct(rtwdev, rtwvif); rtw89_mac_port_cfg_rx_sw(rtwdev, rtwvif); rtw89_mac_port_cfg_rx_sync(rtwdev, rtwvif); - rtw89_mac_port_cfg_tx_sw(rtwdev, rtwvif); + rtw89_mac_port_cfg_tx_sw_by_nettype(rtwdev, rtwvif); rtw89_mac_port_cfg_bcn_intv(rtwdev, rtwvif); rtw89_mac_port_cfg_hiq_win(rtwdev, rtwvif); rtw89_mac_port_cfg_hiq_dtim(rtwdev, rtwvif); @@ -4268,7 +4594,7 @@ void rtw89_mac_set_he_obss_narrow_bw_ru(struct rtw89_dev *rtwdev, void rtw89_mac_stop_ap(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif) { - rtw89_mac_port_cfg_func_en(rtwdev, rtwvif, false); + rtw89_mac_port_cfg_func_sw(rtwdev, rtwvif); } int rtw89_mac_add_vif(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif) @@ -4345,8 +4671,10 @@ rtw89_mac_c2h_scanofld_rsp(struct rtw89_dev *rtwdev, struct sk_buff *c2h, switch (reason) { case RTW89_SCAN_LEAVE_CH_NOTIFY: - if (rtw89_is_op_chan(rtwdev, band, chan)) + if (rtw89_is_op_chan(rtwdev, band, chan)) { + rtw89_mac_enable_beacon_for_ap_vifs(rtwdev, false); ieee80211_stop_queues(rtwdev->hw); + } return; case RTW89_SCAN_END_SCAN_NOTIFY: if (rtwvif && rtwvif->scan_req && @@ -4364,6 +4692,7 @@ rtw89_mac_c2h_scanofld_rsp(struct rtw89_dev *rtwdev, struct sk_buff *c2h, if (rtw89_is_op_chan(rtwdev, band, chan)) { rtw89_assign_entity_chan(rtwdev, rtwvif->sub_entity_idx, &rtwdev->scan_info.op_chan); + rtw89_mac_enable_beacon_for_ap_vifs(rtwdev, true); ieee80211_wake_queues(rtwdev->hw); } else { rtw89_chan_create(&new, chan, chan, band, @@ -5623,7 +5952,8 @@ int rtw89_mac_set_hw_muedca_ctrl(struct rtw89_dev *rtwdev, return 0; } -int rtw89_mac_write_xtal_si(struct rtw89_dev *rtwdev, u8 offset, u8 val, u8 mask) +static +int rtw89_mac_write_xtal_si_ax(struct rtw89_dev *rtwdev, u8 offset, u8 val, u8 mask) { u32 val32; int ret; @@ -5645,9 +5975,9 @@ int rtw89_mac_write_xtal_si(struct rtw89_dev *rtwdev, u8 offset, u8 val, u8 mask return 0; } -EXPORT_SYMBOL(rtw89_mac_write_xtal_si); -int rtw89_mac_read_xtal_si(struct rtw89_dev *rtwdev, u8 offset, u8 *val) +static +int rtw89_mac_read_xtal_si_ax(struct rtw89_dev *rtwdev, u8 offset, u8 *val) { u32 val32; int ret; @@ -5670,7 +6000,6 @@ int rtw89_mac_read_xtal_si(struct rtw89_dev *rtwdev, u8 offset, u8 *val) return 0; } -EXPORT_SYMBOL(rtw89_mac_read_xtal_si); static void rtw89_mac_pkt_drop_sta(struct rtw89_dev *rtwdev, struct rtw89_sta *rtwsta) @@ -5720,6 +6049,7 @@ void rtw89_mac_pkt_drop_vif(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif) int rtw89_mac_ptk_drop_by_band_and_wait(struct rtw89_dev *rtwdev, enum rtw89_mac_idx band) { + const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def; struct rtw89_pkt_drop_params params = {0}; bool empty; int i, ret = 0, try_cnt = 3; @@ -5728,7 +6058,7 @@ int rtw89_mac_ptk_drop_by_band_and_wait(struct rtw89_dev *rtwdev, params.sel = RTW89_PKT_DROP_SEL_BAND_ONCE; for (i = 0; i < try_cnt; i++) { - ret = read_poll_timeout(mac_is_txq_empty, empty, empty, 50, + ret = read_poll_timeout(mac->is_txq_empty, empty, empty, 50, 50000, false, rtwdev); if (ret && !RTW89_CHK_FW_FEATURE(NO_PACKET_DROP, &rtwdev->fw)) rtw89_fw_h2c_pkt_drop(rtwdev, ¶ms); @@ -5776,8 +6106,28 @@ const struct rtw89_mac_gen_def rtw89_mac_gen_ax = { B_AX_BFMEE_HE_NDPA_EN, }, + .check_mac_en = rtw89_mac_check_mac_en_ax, + .sys_init = sys_init_ax, + .trx_init = trx_init_ax, + .hci_func_en = rtw89_mac_hci_func_en_ax, + .dmac_func_pre_en = rtw89_mac_dmac_func_pre_en_ax, + .dle_func_en = dle_func_en_ax, + .dle_clk_en = dle_clk_en_ax, .bf_assoc = rtw89_mac_bf_assoc_ax, + .typ_fltr_opt = rtw89_mac_typ_fltr_opt_ax, + + .dle_mix_cfg = dle_mix_cfg_ax, + .chk_dle_rdy = chk_dle_rdy_ax, + .dle_buf_req = dle_buf_req_ax, + .hfc_func_en = hfc_func_en_ax, + .hfc_h2c_cfg = hfc_h2c_cfg_ax, + .hfc_mix_cfg = hfc_mix_cfg_ax, + .hfc_get_mix_info = hfc_get_mix_info_ax, + .wde_quota_cfg = wde_quota_cfg_ax, + .ple_quota_cfg = ple_quota_cfg_ax, + .set_cpuio = set_cpuio_ax, + .disable_cpu = rtw89_mac_disable_cpu_ax, .fwdl_enable_wcpu = rtw89_mac_enable_cpu_ax, .fwdl_get_status = rtw89_fw_get_rdy_ax, @@ -5787,5 +6137,13 @@ const struct rtw89_mac_gen_def rtw89_mac_gen_ax = { .cnv_efuse_state = rtw89_cnv_efuse_state_ax, .get_txpwr_cr = rtw89_mac_get_txpwr_cr_ax, + + .write_xtal_si = rtw89_mac_write_xtal_si_ax, + .read_xtal_si = rtw89_mac_read_xtal_si_ax, + + .dump_qta_lost = rtw89_mac_dump_qta_lost_ax, + .dump_err_status = rtw89_mac_dump_err_status_ax, + + .is_txq_empty = mac_is_txq_empty_ax, }; EXPORT_SYMBOL(rtw89_mac_gen_ax); diff --git a/drivers/net/wireless/realtek/rtw89/mac.h b/drivers/net/wireless/realtek/rtw89/mac.h index f47a42387a6a6c..ed98b49809a463 100644 --- a/drivers/net/wireless/realtek/rtw89/mac.h +++ b/drivers/net/wireless/realtek/rtw89/mac.h @@ -537,6 +537,9 @@ enum rtw89_mac_bf_rrsc_rate { #define B_CMAC1_MGQ_NO_PWRSAV BIT(11) #define B_CMAC1_CPUMGQ BIT(12) +#define B_CMAC0_MGQ_NORMAL_BE BIT(2) +#define B_CMAC1_MGQ_NORMAL_BE BIT(30) + #define QEMP_ACQ_GRP_MACID_NUM 8 #define QEMP_ACQ_GRP_QSEL_SH 4 #define QEMP_ACQ_GRP_QSEL_MASK 0xF @@ -650,6 +653,22 @@ struct rtw89_mac_dle_dfi_qempty { u32 qempty; }; +enum rtw89_mac_dle_rsvd_qt_type { + DLE_RSVD_QT_MPDU_INFO, + DLE_RSVD_QT_B0_CSI, + DLE_RSVD_QT_B1_CSI, + DLE_RSVD_QT_B0_LMR, + DLE_RSVD_QT_B1_LMR, + DLE_RSVD_QT_B0_FTM, + DLE_RSVD_QT_B1_FTM, +}; + +struct rtw89_mac_dle_rsvd_qt_cfg { + u16 pktid; + u16 pg_num; + u32 size; +}; + enum rtw89_mac_error_scenario { RTW89_RXI300_ERROR = 1, RTW89_WCPU_CPU_EXCEPTION = 2, @@ -818,27 +837,37 @@ enum mac_ax_err_info { struct rtw89_mac_size_set { const struct rtw89_hfc_prec_cfg hfc_preccfg_pcie; + const struct rtw89_hfc_prec_cfg hfc_prec_cfg_c0; + const struct rtw89_hfc_prec_cfg hfc_prec_cfg_c2; const struct rtw89_dle_size wde_size0; + const struct rtw89_dle_size wde_size0_v1; const struct rtw89_dle_size wde_size4; + const struct rtw89_dle_size wde_size4_v1; const struct rtw89_dle_size wde_size6; const struct rtw89_dle_size wde_size7; const struct rtw89_dle_size wde_size9; const struct rtw89_dle_size wde_size18; const struct rtw89_dle_size wde_size19; const struct rtw89_dle_size ple_size0; + const struct rtw89_dle_size ple_size0_v1; + const struct rtw89_dle_size ple_size3_v1; const struct rtw89_dle_size ple_size4; const struct rtw89_dle_size ple_size6; const struct rtw89_dle_size ple_size8; const struct rtw89_dle_size ple_size18; const struct rtw89_dle_size ple_size19; const struct rtw89_wde_quota wde_qt0; + const struct rtw89_wde_quota wde_qt0_v1; const struct rtw89_wde_quota wde_qt4; const struct rtw89_wde_quota wde_qt6; const struct rtw89_wde_quota wde_qt7; const struct rtw89_wde_quota wde_qt17; const struct rtw89_wde_quota wde_qt18; + const struct rtw89_ple_quota ple_qt0; + const struct rtw89_ple_quota ple_qt1; const struct rtw89_ple_quota ple_qt4; const struct rtw89_ple_quota ple_qt5; + const struct rtw89_ple_quota ple_qt9; const struct rtw89_ple_quota ple_qt13; const struct rtw89_ple_quota ple_qt18; const struct rtw89_ple_quota ple_qt44; @@ -849,6 +878,10 @@ struct rtw89_mac_size_set { const struct rtw89_ple_quota ple_qt_52a_wow; const struct rtw89_ple_quota ple_qt_52b_wow; const struct rtw89_ple_quota ple_qt_51b_wow; + const struct rtw89_rsvd_quota ple_rsvd_qt0; + const struct rtw89_rsvd_quota ple_rsvd_qt1; + const struct rtw89_dle_rsvd_size rsvd0_size0; + const struct rtw89_dle_rsvd_size rsvd1_size0; }; extern const struct rtw89_mac_size_set rtw89_mac_size; @@ -865,9 +898,39 @@ struct rtw89_mac_gen_def { struct rtw89_reg_def muedca_ctrl; struct rtw89_reg_def bfee_ctrl; + int (*check_mac_en)(struct rtw89_dev *rtwdev, u8 band, + enum rtw89_mac_hwmod_sel sel); + int (*sys_init)(struct rtw89_dev *rtwdev); + int (*trx_init)(struct rtw89_dev *rtwdev); + void (*hci_func_en)(struct rtw89_dev *rtwdev); + void (*dmac_func_pre_en)(struct rtw89_dev *rtwdev); + void (*dle_func_en)(struct rtw89_dev *rtwdev, bool enable); + void (*dle_clk_en)(struct rtw89_dev *rtwdev, bool enable); void (*bf_assoc)(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif, struct ieee80211_sta *sta); + int (*typ_fltr_opt)(struct rtw89_dev *rtwdev, + enum rtw89_machdr_frame_type type, + enum rtw89_mac_fwd_target fwd_target, + u8 mac_idx); + + int (*dle_mix_cfg)(struct rtw89_dev *rtwdev, const struct rtw89_dle_mem *cfg); + int (*chk_dle_rdy)(struct rtw89_dev *rtwdev, bool wde_or_ple); + int (*dle_buf_req)(struct rtw89_dev *rtwdev, u16 buf_len, bool wd, u16 *pkt_id); + void (*hfc_func_en)(struct rtw89_dev *rtwdev, bool en, bool h2c_en); + void (*hfc_h2c_cfg)(struct rtw89_dev *rtwdev); + void (*hfc_mix_cfg)(struct rtw89_dev *rtwdev); + void (*hfc_get_mix_info)(struct rtw89_dev *rtwdev); + void (*wde_quota_cfg)(struct rtw89_dev *rtwdev, + const struct rtw89_wde_quota *min_cfg, + const struct rtw89_wde_quota *max_cfg, + u16 ext_wde_min_qt_wcpu); + void (*ple_quota_cfg)(struct rtw89_dev *rtwdev, + const struct rtw89_ple_quota *min_cfg, + const struct rtw89_ple_quota *max_cfg); + int (*set_cpuio)(struct rtw89_dev *rtwdev, + struct rtw89_cpuio_ctrl *ctrl_para, bool wd); + void (*disable_cpu)(struct rtw89_dev *rtwdev); int (*fwdl_enable_wcpu)(struct rtw89_dev *rtwdev, u8 boot_reason, bool dlfw, bool include_bb); @@ -880,6 +943,15 @@ struct rtw89_mac_gen_def { bool (*get_txpwr_cr)(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy_idx, u32 reg_base, u32 *cr); + + int (*write_xtal_si)(struct rtw89_dev *rtwdev, u8 offset, u8 val, u8 mask); + int (*read_xtal_si)(struct rtw89_dev *rtwdev, u8 offset, u8 *val); + + void (*dump_qta_lost)(struct rtw89_dev *rtwdev); + void (*dump_err_status)(struct rtw89_dev *rtwdev, + enum mac_ax_err_info err); + + bool (*is_txq_empty)(struct rtw89_dev *rtwdev); }; extern const struct rtw89_mac_gen_def rtw89_mac_gen_ax; @@ -981,10 +1053,31 @@ rtw89_write32_port_set(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif, void rtw89_mac_pwr_off(struct rtw89_dev *rtwdev); int rtw89_mac_partial_init(struct rtw89_dev *rtwdev, bool include_bb); int rtw89_mac_init(struct rtw89_dev *rtwdev); +int rtw89_mac_dle_init(struct rtw89_dev *rtwdev, enum rtw89_qta_mode mode, + enum rtw89_qta_mode ext_mode); +int rtw89_mac_hfc_init(struct rtw89_dev *rtwdev, bool reset, bool en, bool h2c_en); +int rtw89_mac_preload_init(struct rtw89_dev *rtwdev, enum rtw89_mac_idx mac_idx, + enum rtw89_qta_mode mode); +bool rtw89_mac_is_qta_dbcc(struct rtw89_dev *rtwdev, enum rtw89_qta_mode mode); +static inline int rtw89_mac_check_mac_en(struct rtw89_dev *rtwdev, u8 band, - enum rtw89_mac_hwmod_sel sel); + enum rtw89_mac_hwmod_sel sel) +{ + const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def; + + return mac->check_mac_en(rtwdev, band, sel); +} + int rtw89_mac_write_lte(struct rtw89_dev *rtwdev, const u32 offset, u32 val); int rtw89_mac_read_lte(struct rtw89_dev *rtwdev, const u32 offset, u32 *val); +int rtw89_mac_dle_dfi_cfg(struct rtw89_dev *rtwdev, struct rtw89_mac_dle_dfi_ctrl *ctrl); +int rtw89_mac_dle_dfi_quota_cfg(struct rtw89_dev *rtwdev, + struct rtw89_mac_dle_dfi_quota *quota); +void rtw89_mac_dump_dmac_err_status(struct rtw89_dev *rtwdev); +int rtw89_mac_dle_dfi_qempty_cfg(struct rtw89_dev *rtwdev, + struct rtw89_mac_dle_dfi_qempty *qempty); +void rtw89_mac_dump_l0_to_l1(struct rtw89_dev *rtwdev, + enum mac_ax_err_info err); int rtw89_mac_add_vif(struct rtw89_dev *rtwdev, struct rtw89_vif *vif); int rtw89_mac_port_update(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif); void rtw89_mac_port_tsf_sync(struct rtw89_dev *rtwdev, @@ -996,6 +1089,7 @@ int rtw89_mac_port_get_tsf(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif, void rtw89_mac_set_he_obss_narrow_bw_ru(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif); void rtw89_mac_stop_ap(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif); +void rtw89_mac_enable_beacon_for_ap_vifs(struct rtw89_dev *rtwdev, bool en); int rtw89_mac_remove_vif(struct rtw89_dev *rtwdev, struct rtw89_vif *vif); int rtw89_mac_enable_bb_rf(struct rtw89_dev *rtwdev); int rtw89_mac_disable_bb_rf(struct rtw89_dev *rtwdev); @@ -1014,6 +1108,23 @@ static inline int rtw89_chip_disable_bb_rf(struct rtw89_dev *rtwdev) return chip->ops->disable_bb_rf(rtwdev); } +static inline int rtw89_chip_reset_bb_rf(struct rtw89_dev *rtwdev) +{ + int ret; + + if (rtwdev->chip->chip_gen != RTW89_CHIP_AX) + return 0; + + ret = rtw89_chip_disable_bb_rf(rtwdev); + if (ret) + return ret; + ret = rtw89_chip_enable_bb_rf(rtwdev); + if (ret) + return ret; + + return 0; +} + u32 rtw89_mac_get_err_status(struct rtw89_dev *rtwdev); int rtw89_mac_set_err_status(struct rtw89_dev *rtwdev, u32 err); bool rtw89_mac_c2h_chk_atomic(struct rtw89_dev *rtwdev, u8 class, u8 func); @@ -1189,6 +1300,7 @@ enum rtw89_mac_xtal_si_offset { #define XTAL_SC_XI_MASK GENMASK(7, 0) XTAL_SI_XTAL_SC_XO = 0x05, #define XTAL_SC_XO_MASK GENMASK(7, 0) + XTAL_SI_XREF_MODE = 0x0B, XTAL_SI_PWR_CUT = 0x10, #define XTAL_SI_SMALL_PWR_CUT BIT(0) #define XTAL_SI_BIG_PWR_CUT BIT(1) @@ -1198,6 +1310,8 @@ enum rtw89_mac_xtal_si_offset { #define XTAL_SI_LDO_LPS GENMASK(6, 4) XTAL_SI_XTAL_XMD_4 = 0x26, #define XTAL_SI_LPS_CAP GENMASK(3, 0) + XTAL_SI_XREF_RF1 = 0x2D, + XTAL_SI_XREF_RF2 = 0x2E, XTAL_SI_CV = 0x41, #define XTAL_SI_ACV_MASK GENMASK(3, 0) XTAL_SI_LOW_ADDR = 0x62, @@ -1225,20 +1339,34 @@ enum rtw89_mac_xtal_si_offset { XTAL_SI_SRAM_CTRL = 0xA1, #define XTAL_SI_SRAM_DIS BIT(1) #define FULL_BIT_MASK GENMASK(7, 0) + XTAL_SI_PLL = 0xE0, + XTAL_SI_PLL_1 = 0xE1, }; -int rtw89_mac_write_xtal_si(struct rtw89_dev *rtwdev, u8 offset, u8 val, u8 mask); -int rtw89_mac_read_xtal_si(struct rtw89_dev *rtwdev, u8 offset, u8 *val); +static inline +int rtw89_mac_write_xtal_si(struct rtw89_dev *rtwdev, u8 offset, u8 val, u8 mask) +{ + const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def; + + return mac->write_xtal_si(rtwdev, offset, val, mask); +} + +static inline +int rtw89_mac_read_xtal_si(struct rtw89_dev *rtwdev, u8 offset, u8 *val) +{ + const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def; + + return mac->read_xtal_si(rtwdev, offset, val); +} + void rtw89_mac_pkt_drop_vif(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif); -int rtw89_mac_dle_buf_req(struct rtw89_dev *rtwdev, u16 buf_len, bool wd, u16 *pkt_id); -int rtw89_mac_set_cpuio(struct rtw89_dev *rtwdev, - struct rtw89_cpuio_ctrl *ctrl_para, bool wd); -int rtw89_mac_typ_fltr_opt(struct rtw89_dev *rtwdev, - enum rtw89_machdr_frame_type type, - enum rtw89_mac_fwd_target fwd_target, u8 mac_idx); int rtw89_mac_resize_ple_rx_quota(struct rtw89_dev *rtwdev, bool wow); int rtw89_mac_ptk_drop_by_band_and_wait(struct rtw89_dev *rtwdev, enum rtw89_mac_idx band); void rtw89_mac_hw_mgnt_sec(struct rtw89_dev *rtwdev, bool wow); +int rtw89_mac_dle_quota_change(struct rtw89_dev *rtwdev, enum rtw89_qta_mode mode); +int rtw89_mac_get_dle_rsvd_qt_cfg(struct rtw89_dev *rtwdev, + enum rtw89_mac_dle_rsvd_qt_type type, + struct rtw89_mac_dle_rsvd_qt_cfg *cfg); #endif diff --git a/drivers/net/wireless/realtek/rtw89/mac80211.c b/drivers/net/wireless/realtek/rtw89/mac80211.c index 31d1f78916751e..93889d2fface11 100644 --- a/drivers/net/wireless/realtek/rtw89/mac80211.c +++ b/drivers/net/wireless/realtek/rtw89/mac80211.c @@ -226,6 +226,7 @@ static void rtw89_ops_configure_filter(struct ieee80211_hw *hw, { struct rtw89_dev *rtwdev = hw->priv; const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def; + u32 rx_fltr; mutex_lock(&rtwdev->mutex); rtw89_leave_ps_mode(rtwdev); @@ -272,16 +273,29 @@ static void rtw89_ops_configure_filter(struct ieee80211_hw *hw, } } + rx_fltr = rtwdev->hal.rx_fltr; + + /* mac80211 doesn't configure filter when HW scan, driver need to + * set by itself. However, during P2P scan might have configure + * filter to overwrite filter that HW scan needed, so we need to + * check scan and append related filter + */ + if (rtwdev->scanning) { + rx_fltr &= ~B_AX_A_BCN_CHK_EN; + rx_fltr &= ~B_AX_A_BC; + rx_fltr &= ~B_AX_A_A1_MATCH; + } + rtw89_write32_mask(rtwdev, rtw89_mac_reg_by_idx(rtwdev, mac->rx_fltr, RTW89_MAC_0), B_AX_RX_FLTR_CFG_MASK, - rtwdev->hal.rx_fltr); + rx_fltr); if (!rtwdev->dbcc_en) goto out; rtw89_write32_mask(rtwdev, rtw89_mac_reg_by_idx(rtwdev, mac->rx_fltr, RTW89_MAC_1), B_AX_RX_FLTR_CFG_MASK, - rtwdev->hal.rx_fltr); + rx_fltr); out: mutex_unlock(&rtwdev->mutex); @@ -477,6 +491,9 @@ static int rtw89_ops_start_ap(struct ieee80211_hw *hw, return -EOPNOTSUPP; } + if (rtwdev->scanning) + rtw89_hw_scan_abort(rtwdev, rtwdev->scan_info.scanning_vif); + ether_addr_copy(rtwvif->bssid, vif->bss_conf.bssid); rtw89_cam_bssid_changed(rtwdev, rtwvif); rtw89_mac_port_update(rtwdev, rtwvif); diff --git a/drivers/net/wireless/realtek/rtw89/mac_be.c b/drivers/net/wireless/realtek/rtw89/mac_be.c index 1c607316f65258..fa3f5ef289cb5a 100644 --- a/drivers/net/wireless/realtek/rtw89/mac_be.c +++ b/drivers/net/wireless/realtek/rtw89/mac_be.c @@ -57,6 +57,364 @@ static const struct rtw89_port_reg rtw89_port_base_be = { R_BE_PORT_HGQ_WINDOW_CFG + 3}, }; +static int rtw89_mac_check_mac_en_be(struct rtw89_dev *rtwdev, u8 mac_idx, + enum rtw89_mac_hwmod_sel sel) +{ + if (sel == RTW89_DMAC_SEL && + test_bit(RTW89_FLAG_DMAC_FUNC, rtwdev->flags)) + return 0; + if (sel == RTW89_CMAC_SEL && mac_idx == RTW89_MAC_0 && + test_bit(RTW89_FLAG_CMAC0_FUNC, rtwdev->flags)) + return 0; + if (sel == RTW89_CMAC_SEL && mac_idx == RTW89_MAC_1 && + test_bit(RTW89_FLAG_CMAC1_FUNC, rtwdev->flags)) + return 0; + + return -EFAULT; +} + +static void hfc_get_mix_info_be(struct rtw89_dev *rtwdev) +{ + struct rtw89_hfc_param *param = &rtwdev->mac.hfc_param; + struct rtw89_hfc_prec_cfg *prec_cfg = ¶m->prec_cfg; + struct rtw89_hfc_pub_cfg *pub_cfg = ¶m->pub_cfg; + struct rtw89_hfc_pub_info *info = ¶m->pub_info; + u32 val; + + val = rtw89_read32(rtwdev, R_BE_PUB_PAGE_INFO1); + info->g0_used = u32_get_bits(val, B_BE_G0_USE_PG_MASK); + info->g1_used = u32_get_bits(val, B_BE_G1_USE_PG_MASK); + + val = rtw89_read32(rtwdev, R_BE_PUB_PAGE_INFO3); + info->g0_aval = u32_get_bits(val, B_BE_G0_AVAL_PG_MASK); + info->g1_aval = u32_get_bits(val, B_BE_G1_AVAL_PG_MASK); + info->pub_aval = u32_get_bits(rtw89_read32(rtwdev, R_BE_PUB_PAGE_INFO2), + B_BE_PUB_AVAL_PG_MASK); + info->wp_aval = u32_get_bits(rtw89_read32(rtwdev, R_BE_WP_PAGE_INFO1), + B_BE_WP_AVAL_PG_MASK); + + val = rtw89_read32(rtwdev, R_BE_HCI_FC_CTRL); + param->en = !!(val & B_BE_HCI_FC_EN); + param->h2c_en = !!(val & B_BE_HCI_FC_CH12_EN); + param->mode = u32_get_bits(val, B_BE_HCI_FC_MODE_MASK); + prec_cfg->ch011_full_cond = u32_get_bits(val, B_BE_HCI_FC_WD_FULL_COND_MASK); + prec_cfg->h2c_full_cond = u32_get_bits(val, B_BE_HCI_FC_CH12_FULL_COND_MASK); + prec_cfg->wp_ch07_full_cond = + u32_get_bits(val, B_BE_HCI_FC_WP_CH07_FULL_COND_MASK); + prec_cfg->wp_ch811_full_cond = + u32_get_bits(val, B_BE_HCI_FC_WP_CH811_FULL_COND_MASK); + + val = rtw89_read32(rtwdev, R_BE_CH_PAGE_CTRL); + prec_cfg->ch011_prec = u32_get_bits(val, B_BE_PREC_PAGE_CH011_V1_MASK); + prec_cfg->h2c_prec = u32_get_bits(val, B_BE_PREC_PAGE_CH12_V1_MASK); + + val = rtw89_read32(rtwdev, R_BE_PUB_PAGE_CTRL2); + pub_cfg->pub_max = u32_get_bits(val, B_BE_PUBPG_ALL_MASK); + + val = rtw89_read32(rtwdev, R_BE_WP_PAGE_CTRL1); + prec_cfg->wp_ch07_prec = u32_get_bits(val, B_BE_PREC_PAGE_WP_CH07_MASK); + prec_cfg->wp_ch811_prec = u32_get_bits(val, B_BE_PREC_PAGE_WP_CH811_MASK); + + val = rtw89_read32(rtwdev, R_BE_WP_PAGE_CTRL2); + pub_cfg->wp_thrd = u32_get_bits(val, B_BE_WP_THRD_MASK); + + val = rtw89_read32(rtwdev, R_BE_PUB_PAGE_CTRL1); + pub_cfg->grp0 = u32_get_bits(val, B_BE_PUBPG_G0_MASK); + pub_cfg->grp1 = u32_get_bits(val, B_BE_PUBPG_G1_MASK); +} + +static void hfc_h2c_cfg_be(struct rtw89_dev *rtwdev) +{ + struct rtw89_hfc_param *param = &rtwdev->mac.hfc_param; + const struct rtw89_hfc_prec_cfg *prec_cfg = ¶m->prec_cfg; + u32 val; + + val = u32_encode_bits(prec_cfg->h2c_prec, B_BE_PREC_PAGE_CH12_V1_MASK); + rtw89_write32(rtwdev, R_BE_CH_PAGE_CTRL, val); +} + +static void hfc_mix_cfg_be(struct rtw89_dev *rtwdev) +{ + struct rtw89_hfc_param *param = &rtwdev->mac.hfc_param; + const struct rtw89_hfc_prec_cfg *prec_cfg = ¶m->prec_cfg; + const struct rtw89_hfc_pub_cfg *pub_cfg = ¶m->pub_cfg; + u32 val; + + val = u32_encode_bits(prec_cfg->ch011_prec, B_BE_PREC_PAGE_CH011_V1_MASK) | + u32_encode_bits(prec_cfg->h2c_prec, B_BE_PREC_PAGE_CH12_V1_MASK); + rtw89_write32(rtwdev, R_BE_CH_PAGE_CTRL, val); + + val = u32_encode_bits(pub_cfg->pub_max, B_BE_PUBPG_ALL_MASK); + rtw89_write32(rtwdev, R_BE_PUB_PAGE_CTRL2, val); + + val = u32_encode_bits(prec_cfg->wp_ch07_prec, B_BE_PREC_PAGE_WP_CH07_MASK) | + u32_encode_bits(prec_cfg->wp_ch811_prec, B_BE_PREC_PAGE_WP_CH811_MASK); + rtw89_write32(rtwdev, R_BE_WP_PAGE_CTRL1, val); + + val = u32_replace_bits(rtw89_read32(rtwdev, R_BE_HCI_FC_CTRL), + param->mode, B_BE_HCI_FC_MODE_MASK); + val = u32_replace_bits(val, prec_cfg->ch011_full_cond, + B_BE_HCI_FC_WD_FULL_COND_MASK); + val = u32_replace_bits(val, prec_cfg->h2c_full_cond, + B_BE_HCI_FC_CH12_FULL_COND_MASK); + val = u32_replace_bits(val, prec_cfg->wp_ch07_full_cond, + B_BE_HCI_FC_WP_CH07_FULL_COND_MASK); + val = u32_replace_bits(val, prec_cfg->wp_ch811_full_cond, + B_BE_HCI_FC_WP_CH811_FULL_COND_MASK); + rtw89_write32(rtwdev, R_BE_HCI_FC_CTRL, val); +} + +static void hfc_func_en_be(struct rtw89_dev *rtwdev, bool en, bool h2c_en) +{ + struct rtw89_hfc_param *param = &rtwdev->mac.hfc_param; + u32 val; + + val = rtw89_read32(rtwdev, R_BE_HCI_FC_CTRL); + param->en = en; + param->h2c_en = h2c_en; + val = en ? (val | B_BE_HCI_FC_EN) : (val & ~B_BE_HCI_FC_EN); + val = h2c_en ? (val | B_BE_HCI_FC_CH12_EN) : + (val & ~B_BE_HCI_FC_CH12_EN); + rtw89_write32(rtwdev, R_BE_HCI_FC_CTRL, val); +} + +static void dle_func_en_be(struct rtw89_dev *rtwdev, bool enable) +{ + if (enable) + rtw89_write32_set(rtwdev, R_BE_DMAC_FUNC_EN, + B_BE_DLE_WDE_EN | B_BE_DLE_PLE_EN); + else + rtw89_write32_clr(rtwdev, R_BE_DMAC_FUNC_EN, + B_BE_DLE_WDE_EN | B_BE_DLE_PLE_EN); +} + +static void dle_clk_en_be(struct rtw89_dev *rtwdev, bool enable) +{ + if (enable) + rtw89_write32_set(rtwdev, R_BE_DMAC_CLK_EN, + B_BE_DLE_WDE_CLK_EN | B_BE_DLE_PLE_CLK_EN); + else + rtw89_write32_clr(rtwdev, R_BE_DMAC_CLK_EN, + B_BE_DLE_WDE_CLK_EN | B_BE_DLE_PLE_CLK_EN); +} + +static int dle_mix_cfg_be(struct rtw89_dev *rtwdev, const struct rtw89_dle_mem *cfg) +{ + const struct rtw89_dle_size *wde_size_cfg, *ple_size_cfg; + u32 bound; + u32 val; + + wde_size_cfg = cfg->wde_size; + ple_size_cfg = cfg->ple_size; + + val = rtw89_read32(rtwdev, R_BE_WDE_PKTBUF_CFG); + + switch (wde_size_cfg->pge_size) { + default: + case RTW89_WDE_PG_64: + val = u32_replace_bits(val, S_AX_WDE_PAGE_SEL_64, + B_BE_WDE_PAGE_SEL_MASK); + break; + case RTW89_WDE_PG_128: + val = u32_replace_bits(val, S_AX_WDE_PAGE_SEL_128, + B_BE_WDE_PAGE_SEL_MASK); + break; + case RTW89_WDE_PG_256: + rtw89_err(rtwdev, "[ERR]WDE DLE doesn't support 256 byte!\n"); + return -EINVAL; + } + + bound = wde_size_cfg->srt_ofst / DLE_BOUND_UNIT; + val = u32_replace_bits(val, bound, B_BE_WDE_START_BOUND_MASK); + val = u32_replace_bits(val, wde_size_cfg->lnk_pge_num, + B_BE_WDE_FREE_PAGE_NUM_MASK); + rtw89_write32(rtwdev, R_BE_WDE_PKTBUF_CFG, val); + + val = rtw89_read32(rtwdev, R_BE_PLE_PKTBUF_CFG); + + switch (ple_size_cfg->pge_size) { + default: + case RTW89_PLE_PG_64: + rtw89_err(rtwdev, "[ERR]PLE DLE doesn't support 64 byte!\n"); + return -EINVAL; + case RTW89_PLE_PG_128: + val = u32_replace_bits(val, S_AX_PLE_PAGE_SEL_128, + B_BE_PLE_PAGE_SEL_MASK); + break; + case RTW89_PLE_PG_256: + val = u32_replace_bits(val, S_AX_PLE_PAGE_SEL_256, + B_BE_PLE_PAGE_SEL_MASK); + break; + } + + bound = ple_size_cfg->srt_ofst / DLE_BOUND_UNIT; + val = u32_replace_bits(val, bound, B_BE_PLE_START_BOUND_MASK); + val = u32_replace_bits(val, ple_size_cfg->lnk_pge_num, + B_BE_PLE_FREE_PAGE_NUM_MASK); + rtw89_write32(rtwdev, R_BE_PLE_PKTBUF_CFG, val); + + return 0; +} + +static int chk_dle_rdy_be(struct rtw89_dev *rtwdev, bool wde_or_ple) +{ + u32 reg, mask; + u32 ini; + + if (wde_or_ple) { + reg = R_AX_WDE_INI_STATUS; + mask = WDE_MGN_INI_RDY; + } else { + reg = R_AX_PLE_INI_STATUS; + mask = PLE_MGN_INI_RDY; + } + + return read_poll_timeout(rtw89_read32, ini, (ini & mask) == mask, 1, + 2000, false, rtwdev, reg); +} + +#define INVALID_QT_WCPU U16_MAX +#define SET_QUOTA_VAL(_min_x, _max_x, _module, _idx) \ + do { \ + val = u32_encode_bits(_min_x, B_BE_ ## _module ## _Q ## _idx ## _MIN_SIZE_MASK) | \ + u32_encode_bits(_max_x, B_BE_ ## _module ## _Q ## _idx ## _MAX_SIZE_MASK); \ + rtw89_write32(rtwdev, \ + R_BE_ ## _module ## _QTA ## _idx ## _CFG, \ + val); \ + } while (0) +#define SET_QUOTA(_x, _module, _idx) \ + SET_QUOTA_VAL(min_cfg->_x, max_cfg->_x, _module, _idx) + +static void wde_quota_cfg_be(struct rtw89_dev *rtwdev, + const struct rtw89_wde_quota *min_cfg, + const struct rtw89_wde_quota *max_cfg, + u16 ext_wde_min_qt_wcpu) +{ + u16 min_qt_wcpu = ext_wde_min_qt_wcpu != INVALID_QT_WCPU ? + ext_wde_min_qt_wcpu : min_cfg->wcpu; + u16 max_qt_wcpu = max(max_cfg->wcpu, min_qt_wcpu); + u32 val; + + SET_QUOTA(hif, WDE, 0); + SET_QUOTA_VAL(min_qt_wcpu, max_qt_wcpu, WDE, 1); + SET_QUOTA_VAL(0, 0, WDE, 2); + SET_QUOTA(pkt_in, WDE, 3); + SET_QUOTA(cpu_io, WDE, 4); +} + +static void ple_quota_cfg_be(struct rtw89_dev *rtwdev, + const struct rtw89_ple_quota *min_cfg, + const struct rtw89_ple_quota *max_cfg) +{ + u32 val; + + SET_QUOTA(cma0_tx, PLE, 0); + SET_QUOTA(cma1_tx, PLE, 1); + SET_QUOTA(c2h, PLE, 2); + SET_QUOTA(h2c, PLE, 3); + SET_QUOTA(wcpu, PLE, 4); + SET_QUOTA(mpdu_proc, PLE, 5); + SET_QUOTA(cma0_dma, PLE, 6); + SET_QUOTA(cma1_dma, PLE, 7); + SET_QUOTA(bb_rpt, PLE, 8); + SET_QUOTA(wd_rel, PLE, 9); + SET_QUOTA(cpu_io, PLE, 10); + SET_QUOTA(tx_rpt, PLE, 11); + SET_QUOTA(h2d, PLE, 12); +} + +static void rtw89_mac_hci_func_en_be(struct rtw89_dev *rtwdev) +{ + rtw89_write32_set(rtwdev, R_BE_HCI_FUNC_EN, B_BE_HCI_TXDMA_EN | + B_BE_HCI_RXDMA_EN); +} + +static void rtw89_mac_dmac_func_pre_en_be(struct rtw89_dev *rtwdev) +{ + u32 val; + + val = rtw89_read32(rtwdev, R_BE_HAXI_INIT_CFG1); + + switch (rtwdev->hci.type) { + case RTW89_HCI_TYPE_PCIE: + val = u32_replace_bits(val, S_BE_DMA_MOD_PCIE_NO_DATA_CPU, + B_BE_DMA_MODE_MASK); + break; + case RTW89_HCI_TYPE_USB: + val = u32_replace_bits(val, S_BE_DMA_MOD_USB, B_BE_DMA_MODE_MASK); + val = (val & ~B_BE_STOP_AXI_MST) | B_BE_TXDMA_EN | B_BE_RXDMA_EN; + break; + case RTW89_HCI_TYPE_SDIO: + val = u32_replace_bits(val, S_BE_DMA_MOD_SDIO, B_BE_DMA_MODE_MASK); + val = (val & ~B_BE_STOP_AXI_MST) | B_BE_TXDMA_EN | B_BE_RXDMA_EN; + break; + default: + return; + } + + rtw89_write32(rtwdev, R_BE_HAXI_INIT_CFG1, val); + + rtw89_write32_clr(rtwdev, R_BE_HAXI_DMA_STOP1, + B_BE_STOP_CH0 | B_BE_STOP_CH1 | B_BE_STOP_CH2 | + B_BE_STOP_CH3 | B_BE_STOP_CH4 | B_BE_STOP_CH5 | + B_BE_STOP_CH6 | B_BE_STOP_CH7 | B_BE_STOP_CH8 | + B_BE_STOP_CH9 | B_BE_STOP_CH10 | B_BE_STOP_CH11 | + B_BE_STOP_CH12 | B_BE_STOP_CH13 | B_BE_STOP_CH14); + + rtw89_write32_set(rtwdev, R_BE_DMAC_TABLE_CTRL, B_BE_DMAC_ADDR_MODE); +} + +static +int rtw89_mac_write_xtal_si_be(struct rtw89_dev *rtwdev, u8 offset, u8 val, u8 mask) +{ + u32 val32; + int ret; + + val32 = u32_encode_bits(offset, B_BE_WL_XTAL_SI_ADDR_MASK) | + u32_encode_bits(val, B_BE_WL_XTAL_SI_DATA_MASK) | + u32_encode_bits(mask, B_BE_WL_XTAL_SI_BITMASK_MASK) | + u32_encode_bits(XTAL_SI_NORMAL_WRITE, B_BE_WL_XTAL_SI_MODE_MASK) | + u32_encode_bits(0, B_BE_WL_XTAL_SI_CHIPID_MASK) | + B_BE_WL_XTAL_SI_CMD_POLL; + rtw89_write32(rtwdev, R_BE_WLAN_XTAL_SI_CTRL, val32); + + ret = read_poll_timeout(rtw89_read32, val32, !(val32 & B_BE_WL_XTAL_SI_CMD_POLL), + 50, 50000, false, rtwdev, R_BE_WLAN_XTAL_SI_CTRL); + if (ret) { + rtw89_warn(rtwdev, "xtal si not ready(W): offset=%x val=%x mask=%x\n", + offset, val, mask); + return ret; + } + + return 0; +} + +static +int rtw89_mac_read_xtal_si_be(struct rtw89_dev *rtwdev, u8 offset, u8 *val) +{ + u32 val32; + int ret; + + val32 = u32_encode_bits(offset, B_BE_WL_XTAL_SI_ADDR_MASK) | + u32_encode_bits(0x0, B_BE_WL_XTAL_SI_DATA_MASK) | + u32_encode_bits(0x0, B_BE_WL_XTAL_SI_BITMASK_MASK) | + u32_encode_bits(XTAL_SI_NORMAL_READ, B_BE_WL_XTAL_SI_MODE_MASK) | + u32_encode_bits(0, B_BE_WL_XTAL_SI_CHIPID_MASK) | + B_BE_WL_XTAL_SI_CMD_POLL; + rtw89_write32(rtwdev, R_BE_WLAN_XTAL_SI_CTRL, val32); + + ret = read_poll_timeout(rtw89_read32, val32, !(val32 & B_BE_WL_XTAL_SI_CMD_POLL), + 50, 50000, false, rtwdev, R_BE_WLAN_XTAL_SI_CTRL); + if (ret) { + rtw89_warn(rtwdev, "xtal si not ready(R): offset=%x\n", offset); + return ret; + } + + *val = rtw89_read8(rtwdev, R_BE_WLAN_XTAL_SI_CTRL + 1); + + return 0; +} + static void rtw89_mac_disable_cpu_be(struct rtw89_dev *rtwdev) { u32 val32; @@ -93,8 +451,6 @@ static int wcpu_on(struct rtw89_dev *rtwdev, u8 boot_reason, bool dlfw) u32 val32; int ret; - rtw89_write32_set(rtwdev, R_BE_UDM0, B_BE_UDM0_DBG_MODE_CTRL); - val32 = rtw89_read32(rtwdev, R_BE_HALT_C2H); if (val32) { rtw89_warn(rtwdev, "[SER] AON L2 Debug register not empty before Boot.\n"); @@ -118,6 +474,10 @@ static int wcpu_on(struct rtw89_dev *rtwdev, u8 boot_reason, bool dlfw) rtw89_write32(rtwdev, R_BE_HALT_H2C_CTRL, 0); rtw89_write32(rtwdev, R_BE_HALT_C2H_CTRL, 0); + val32 = rtw89_read32(rtwdev, R_BE_HISR0); + rtw89_write32(rtwdev, R_BE_HISR0, B_BE_HALT_C2H_INT); + rtw89_debug(rtwdev, RTW89_DBG_SER, "HISR0=0x%x\n", val32); + rtw89_write32_set(rtwdev, R_BE_SYS_CLK_CTRL, B_BE_CPU_CLK_EN); rtw89_write32_clr(rtwdev, R_BE_SYS_CFG5, B_BE_WDT_WAKE_PCIE_EN | B_BE_WDT_WAKE_USB_EN); @@ -206,6 +566,90 @@ static int rtw89_fwdl_check_path_ready_be(struct rtw89_dev *rtwdev, rtwdev, R_BE_WCPU_FW_CTRL); } +static int dle_buf_req_be(struct rtw89_dev *rtwdev, u16 buf_len, bool wd, u16 *pkt_id) +{ + u32 val, reg; + int ret; + + reg = wd ? R_BE_WD_BUF_REQ : R_BE_PL_BUF_REQ; + val = buf_len; + val |= B_BE_WD_BUF_REQ_EXEC; + rtw89_write32(rtwdev, reg, val); + + reg = wd ? R_BE_WD_BUF_STATUS : R_BE_PL_BUF_STATUS; + + ret = read_poll_timeout(rtw89_read32, val, val & B_BE_WD_BUF_STAT_DONE, + 1, 2000, false, rtwdev, reg); + if (ret) + return ret; + + *pkt_id = u32_get_bits(val, B_BE_WD_BUF_STAT_PKTID_MASK); + if (*pkt_id == S_WD_BUF_STAT_PKTID_INVALID) + return -ENOENT; + + return 0; +} + +static int set_cpuio_be(struct rtw89_dev *rtwdev, + struct rtw89_cpuio_ctrl *ctrl_para, bool wd) +{ + u32 val_op0, val_op1, val_op2, val_op3; + u32 val, cmd_type, reg; + int ret; + + cmd_type = ctrl_para->cmd_type; + + reg = wd ? R_BE_WD_CPUQ_OP_3 : R_BE_PL_CPUQ_OP_3; + val_op3 = u32_replace_bits(0, ctrl_para->start_pktid, + B_BE_WD_CPUQ_OP_STRT_PKTID_MASK); + val_op3 = u32_replace_bits(val_op3, ctrl_para->end_pktid, + B_BE_WD_CPUQ_OP_END_PKTID_MASK); + rtw89_write32(rtwdev, reg, val_op3); + + reg = wd ? R_BE_WD_CPUQ_OP_1 : R_BE_PL_CPUQ_OP_1; + val_op1 = u32_replace_bits(0, ctrl_para->src_pid, + B_BE_WD_CPUQ_OP_SRC_PID_MASK); + val_op1 = u32_replace_bits(val_op1, ctrl_para->src_qid, + B_BE_WD_CPUQ_OP_SRC_QID_MASK); + val_op1 = u32_replace_bits(val_op1, ctrl_para->macid, + B_BE_WD_CPUQ_OP_SRC_MACID_MASK); + rtw89_write32(rtwdev, reg, val_op1); + + reg = wd ? R_BE_WD_CPUQ_OP_2 : R_BE_PL_CPUQ_OP_2; + val_op2 = u32_replace_bits(0, ctrl_para->dst_pid, + B_BE_WD_CPUQ_OP_DST_PID_MASK); + val_op2 = u32_replace_bits(val_op2, ctrl_para->dst_qid, + B_BE_WD_CPUQ_OP_DST_QID_MASK); + val_op2 = u32_replace_bits(val_op2, ctrl_para->macid, + B_BE_WD_CPUQ_OP_DST_MACID_MASK); + rtw89_write32(rtwdev, reg, val_op2); + + reg = wd ? R_BE_WD_CPUQ_OP_0 : R_BE_PL_CPUQ_OP_0; + val_op0 = u32_replace_bits(0, cmd_type, + B_BE_WD_CPUQ_OP_CMD_TYPE_MASK); + val_op0 = u32_replace_bits(val_op0, ctrl_para->pkt_num, + B_BE_WD_CPUQ_OP_PKTNUM_MASK); + val_op0 |= B_BE_WD_CPUQ_OP_EXEC; + rtw89_write32(rtwdev, reg, val_op0); + + reg = wd ? R_BE_WD_CPUQ_OP_STATUS : R_BE_PL_CPUQ_OP_STATUS; + + ret = read_poll_timeout(rtw89_read32, val, val & B_BE_WD_CPUQ_OP_STAT_DONE, + 1, 2000, false, rtwdev, reg); + if (ret) { + rtw89_err(rtwdev, "[ERR]set cpuio wd timeout\n"); + rtw89_err(rtwdev, "[ERR]op_0=0x%X, op_1=0x%X, op_2=0x%X\n", + val_op0, val_op1, val_op2); + return ret; + } + + if (cmd_type == CPUIO_OP_CMD_GET_NEXT_PID || + cmd_type == CPUIO_OP_CMD_GET_1ST_PID) + ctrl_para->pktid = u32_get_bits(val, B_BE_WD_CPUQ_OP_PKTID_MASK); + + return 0; +} + static bool rtw89_mac_get_txpwr_cr_be(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy_idx, u32 reg_base, u32 *cr) @@ -405,6 +849,299 @@ static void rtw89_mac_bf_assoc_be(struct rtw89_dev *rtwdev, } } +static void dump_err_status_dispatcher_be(struct rtw89_dev *rtwdev) +{ + rtw89_info(rtwdev, "R_BE_DISP_HOST_IMR=0x%08x ", + rtw89_read32(rtwdev, R_BE_DISP_HOST_IMR)); + rtw89_info(rtwdev, "R_BE_DISP_ERROR_ISR1=0x%08x\n", + rtw89_read32(rtwdev, R_BE_DISP_ERROR_ISR1)); + rtw89_info(rtwdev, "R_BE_DISP_CPU_IMR=0x%08x ", + rtw89_read32(rtwdev, R_BE_DISP_CPU_IMR)); + rtw89_info(rtwdev, "R_BE_DISP_ERROR_ISR2=0x%08x\n", + rtw89_read32(rtwdev, R_BE_DISP_ERROR_ISR2)); + rtw89_info(rtwdev, "R_BE_DISP_OTHER_IMR=0x%08x ", + rtw89_read32(rtwdev, R_BE_DISP_OTHER_IMR)); + rtw89_info(rtwdev, "R_BE_DISP_ERROR_ISR0=0x%08x\n", + rtw89_read32(rtwdev, R_BE_DISP_ERROR_ISR0)); +} + +static void rtw89_mac_dump_qta_lost_be(struct rtw89_dev *rtwdev) +{ + struct rtw89_mac_dle_dfi_qempty qempty; + struct rtw89_mac_dle_dfi_quota quota; + struct rtw89_mac_dle_dfi_ctrl ctrl; + u32 val, not_empty, i; + int ret; + + qempty.dle_type = DLE_CTRL_TYPE_PLE; + qempty.grpsel = 0; + qempty.qempty = ~(u32)0; + ret = rtw89_mac_dle_dfi_qempty_cfg(rtwdev, &qempty); + if (ret) + rtw89_warn(rtwdev, "%s: query DLE fail\n", __func__); + else + rtw89_info(rtwdev, "DLE group0 empty: 0x%x\n", qempty.qempty); + + for (not_empty = ~qempty.qempty, i = 0; not_empty != 0; not_empty >>= 1, i++) { + if (!(not_empty & BIT(0))) + continue; + ctrl.type = DLE_CTRL_TYPE_PLE; + ctrl.target = DLE_DFI_TYPE_QLNKTBL; + ctrl.addr = (QLNKTBL_ADDR_INFO_SEL_0 ? QLNKTBL_ADDR_INFO_SEL : 0) | + u32_encode_bits(i, QLNKTBL_ADDR_TBL_IDX_MASK); + ret = rtw89_mac_dle_dfi_cfg(rtwdev, &ctrl); + if (ret) + rtw89_warn(rtwdev, "%s: query DLE fail\n", __func__); + else + rtw89_info(rtwdev, "qidx%d pktcnt = %d\n", i, + u32_get_bits(ctrl.out_data, + QLNKTBL_DATA_SEL1_PKT_CNT_MASK)); + } + + quota.dle_type = DLE_CTRL_TYPE_PLE; + quota.qtaid = 6; + ret = rtw89_mac_dle_dfi_quota_cfg(rtwdev, "a); + if (ret) + rtw89_warn(rtwdev, "%s: query DLE fail\n", __func__); + else + rtw89_info(rtwdev, "quota6 rsv/use: 0x%x/0x%x\n", + quota.rsv_pgnum, quota.use_pgnum); + + val = rtw89_read32(rtwdev, R_BE_PLE_QTA6_CFG); + rtw89_info(rtwdev, "[PLE][CMAC0_RX]min_pgnum=0x%x\n", + u32_get_bits(val, B_BE_PLE_Q6_MIN_SIZE_MASK)); + rtw89_info(rtwdev, "[PLE][CMAC0_RX]max_pgnum=0x%x\n", + u32_get_bits(val, B_BE_PLE_Q6_MAX_SIZE_MASK)); + val = rtw89_read32(rtwdev, R_BE_RX_FLTR_OPT); + rtw89_info(rtwdev, "[PLE][CMAC0_RX]B_BE_RX_MPDU_MAX_LEN=0x%x\n", + u32_get_bits(val, B_BE_RX_MPDU_MAX_LEN_MASK)); + rtw89_info(rtwdev, "R_BE_RSP_CHK_SIG=0x%08x\n", + rtw89_read32(rtwdev, R_BE_RSP_CHK_SIG)); + rtw89_info(rtwdev, "R_BE_TRXPTCL_RESP_0=0x%08x\n", + rtw89_read32(rtwdev, R_BE_TRXPTCL_RESP_0)); + + if (!rtw89_mac_check_mac_en(rtwdev, RTW89_MAC_1, RTW89_CMAC_SEL)) { + quota.dle_type = DLE_CTRL_TYPE_PLE; + quota.qtaid = 7; + ret = rtw89_mac_dle_dfi_quota_cfg(rtwdev, "a); + if (ret) + rtw89_warn(rtwdev, "%s: query DLE fail\n", __func__); + else + rtw89_info(rtwdev, "quota7 rsv/use: 0x%x/0x%x\n", + quota.rsv_pgnum, quota.use_pgnum); + + val = rtw89_read32(rtwdev, R_BE_PLE_QTA7_CFG); + rtw89_info(rtwdev, "[PLE][CMAC1_RX]min_pgnum=0x%x\n", + u32_get_bits(val, B_BE_PLE_Q7_MIN_SIZE_MASK)); + rtw89_info(rtwdev, "[PLE][CMAC1_RX]max_pgnum=0x%x\n", + u32_get_bits(val, B_BE_PLE_Q7_MAX_SIZE_MASK)); + val = rtw89_read32(rtwdev, R_BE_RX_FLTR_OPT_C1); + rtw89_info(rtwdev, "[PLE][CMAC1_RX]B_BE_RX_MPDU_MAX_LEN=0x%x\n", + u32_get_bits(val, B_BE_RX_MPDU_MAX_LEN_MASK)); + rtw89_info(rtwdev, "R_BE_RSP_CHK_SIG_C1=0x%08x\n", + rtw89_read32(rtwdev, R_BE_RSP_CHK_SIG_C1)); + rtw89_info(rtwdev, "R_BE_TRXPTCL_RESP_0_C1=0x%08x\n", + rtw89_read32(rtwdev, R_BE_TRXPTCL_RESP_0_C1)); + } + + rtw89_info(rtwdev, "R_BE_DLE_EMPTY0=0x%08x\n", + rtw89_read32(rtwdev, R_BE_DLE_EMPTY0)); + rtw89_info(rtwdev, "R_BE_DLE_EMPTY1=0x%08x\n", + rtw89_read32(rtwdev, R_BE_DLE_EMPTY1)); + + dump_err_status_dispatcher_be(rtwdev); +} + +static void rtw89_mac_dump_cmac_err_status_be(struct rtw89_dev *rtwdev, + u8 band) +{ + u32 offset = 0; + u32 cmac_err; + int ret; + + ret = rtw89_mac_check_mac_en(rtwdev, band, RTW89_CMAC_SEL); + if (ret) { + rtw89_info(rtwdev, "[CMAC] : CMAC%d not enabled\n", band); + return; + } + + if (band) + offset = RTW89_MAC_BE_BAND_REG_OFFSET; + + cmac_err = rtw89_read32(rtwdev, R_BE_CMAC_ERR_ISR + offset); + rtw89_info(rtwdev, "R_BE_CMAC_ERR_ISR [%d]=0x%08x\n", band, + rtw89_read32(rtwdev, R_BE_CMAC_ERR_ISR + offset)); + rtw89_info(rtwdev, "R_BE_CMAC_FUNC_EN [%d]=0x%08x\n", band, + rtw89_read32(rtwdev, R_BE_CMAC_FUNC_EN + offset)); + rtw89_info(rtwdev, "R_BE_CK_EN [%d]=0x%08x\n", band, + rtw89_read32(rtwdev, R_BE_CK_EN + offset)); + + if (cmac_err & B_BE_SCHEDULE_TOP_ERR_IND) { + rtw89_info(rtwdev, "R_BE_SCHEDULE_ERR_IMR [%d]=0x%08x\n", band, + rtw89_read32(rtwdev, R_BE_SCHEDULE_ERR_IMR + offset)); + rtw89_info(rtwdev, "R_BE_SCHEDULE_ERR_ISR [%d]=0x%08x\n", band, + rtw89_read32(rtwdev, R_BE_SCHEDULE_ERR_ISR + offset)); + } + + if (cmac_err & B_BE_PTCL_TOP_ERR_IND) { + rtw89_info(rtwdev, "R_BE_PTCL_IMR0 [%d]=0x%08x\n", band, + rtw89_read32(rtwdev, R_BE_PTCL_IMR0 + offset)); + rtw89_info(rtwdev, "R_BE_PTCL_ISR0 [%d]=0x%08x\n", band, + rtw89_read32(rtwdev, R_BE_PTCL_ISR0 + offset)); + rtw89_info(rtwdev, "R_BE_PTCL_IMR1 [%d]=0x%08x\n", band, + rtw89_read32(rtwdev, R_BE_PTCL_IMR1 + offset)); + rtw89_info(rtwdev, "R_BE_PTCL_ISR1 [%d]=0x%08x\n", band, + rtw89_read32(rtwdev, R_BE_PTCL_ISR1 + offset)); + } + + if (cmac_err & B_BE_DMA_TOP_ERR_IND) { + rtw89_info(rtwdev, "R_BE_RX_ERROR_FLAG_IMR [%d]=0x%08x\n", band, + rtw89_read32(rtwdev, R_BE_RX_ERROR_FLAG_IMR + offset)); + rtw89_info(rtwdev, "R_BE_RX_ERROR_FLAG [%d]=0x%08x\n", band, + rtw89_read32(rtwdev, R_BE_RX_ERROR_FLAG + offset)); + rtw89_info(rtwdev, "R_BE_TX_ERROR_FLAG_IMR [%d]=0x%08x\n", band, + rtw89_read32(rtwdev, R_BE_TX_ERROR_FLAG_IMR + offset)); + rtw89_info(rtwdev, "R_BE_TX_ERROR_FLAG [%d]=0x%08x\n", band, + rtw89_read32(rtwdev, R_BE_TX_ERROR_FLAG + offset)); + rtw89_info(rtwdev, "R_BE_RX_ERROR_FLAG_IMR_1 [%d]=0x%08x\n", band, + rtw89_read32(rtwdev, R_BE_RX_ERROR_FLAG_IMR_1 + offset)); + rtw89_info(rtwdev, "R_BE_RX_ERROR_FLAG_1 [%d]=0x%08x\n", band, + rtw89_read32(rtwdev, R_BE_RX_ERROR_FLAG_1 + offset)); + } + + if (cmac_err & B_BE_PHYINTF_ERR_IND) { + rtw89_info(rtwdev, "R_BE_PHYINFO_ERR_IMR [%d]=0x%08x\n", band, + rtw89_read32(rtwdev, R_BE_PHYINFO_ERR_IMR_V1 + offset)); + rtw89_info(rtwdev, "R_BE_PHYINFO_ERR_ISR [%d]=0x%08x\n", band, + rtw89_read32(rtwdev, R_BE_PHYINFO_ERR_ISR + offset)); + } + + if (cmac_err & B_AX_TXPWR_CTRL_ERR_IND) { + rtw89_info(rtwdev, "R_BE_TXPWR_ERR_FLAG [%d]=0x%08x\n", band, + rtw89_read32(rtwdev, R_BE_TXPWR_ERR_FLAG + offset)); + rtw89_info(rtwdev, "R_BE_TXPWR_ERR_IMR [%d]=0x%08x\n", band, + rtw89_read32(rtwdev, R_BE_TXPWR_ERR_IMR + offset)); + } + + if (cmac_err & (B_BE_WMAC_RX_ERR_IND | B_BE_WMAC_TX_ERR_IND | + B_BE_WMAC_RX_IDLETO_IDCT | B_BE_PTCL_TX_IDLETO_IDCT)) { + rtw89_info(rtwdev, "R_BE_DBGSEL_TRXPTCL [%d]=0x%08x\n", band, + rtw89_read32(rtwdev, R_BE_DBGSEL_TRXPTCL + offset)); + rtw89_info(rtwdev, "R_BE_TRXPTCL_ERROR_INDICA_MASK [%d]=0x%08x\n", band, + rtw89_read32(rtwdev, R_BE_TRXPTCL_ERROR_INDICA_MASK + offset)); + rtw89_info(rtwdev, "R_BE_TRXPTCL_ERROR_INDICA [%d]=0x%08x\n", band, + rtw89_read32(rtwdev, R_BE_TRXPTCL_ERROR_INDICA + offset)); + rtw89_info(rtwdev, "R_BE_RX_ERR_IMR [%d]=0x%08x\n", band, + rtw89_read32(rtwdev, R_BE_RX_ERR_IMR + offset)); + rtw89_info(rtwdev, "R_BE_RX_ERR_ISR [%d]=0x%08x\n", band, + rtw89_read32(rtwdev, R_BE_RX_ERR_ISR + offset)); + } + + rtw89_info(rtwdev, "R_BE_CMAC_ERR_IMR [%d]=0x%08x\n", band, + rtw89_read32(rtwdev, R_BE_CMAC_ERR_IMR + offset)); +} + +static void rtw89_mac_dump_err_status_be(struct rtw89_dev *rtwdev, + enum mac_ax_err_info err) +{ + if (err != MAC_AX_ERR_L1_ERR_DMAC && + err != MAC_AX_ERR_L0_PROMOTE_TO_L1 && + err != MAC_AX_ERR_L0_ERR_CMAC0 && + err != MAC_AX_ERR_L0_ERR_CMAC1 && + err != MAC_AX_ERR_RXI300) + return; + + rtw89_info(rtwdev, "--->\nerr=0x%x\n", err); + rtw89_info(rtwdev, "R_BE_SER_DBG_INFO=0x%08x\n", + rtw89_read32(rtwdev, R_BE_SER_DBG_INFO)); + rtw89_info(rtwdev, "R_BE_SER_L0_DBG_CNT=0x%08x\n", + rtw89_read32(rtwdev, R_BE_SER_L0_DBG_CNT)); + rtw89_info(rtwdev, "R_BE_SER_L0_DBG_CNT1=0x%08x\n", + rtw89_read32(rtwdev, R_BE_SER_L0_DBG_CNT1)); + rtw89_info(rtwdev, "R_BE_SER_L0_DBG_CNT2=0x%08x\n", + rtw89_read32(rtwdev, R_BE_SER_L0_DBG_CNT2)); + rtw89_info(rtwdev, "R_BE_SER_L0_DBG_CNT3=0x%08x\n", + rtw89_read32(rtwdev, R_BE_SER_L0_DBG_CNT3)); + if (!rtw89_mac_check_mac_en(rtwdev, RTW89_MAC_1, RTW89_CMAC_SEL)) { + rtw89_info(rtwdev, "R_BE_SER_L0_DBG_CNT_C1=0x%08x\n", + rtw89_read32(rtwdev, R_BE_SER_L0_DBG_CNT_C1)); + rtw89_info(rtwdev, "R_BE_SER_L0_DBG_CNT1_C1=0x%08x\n", + rtw89_read32(rtwdev, R_BE_SER_L0_DBG_CNT1_C1)); + } + rtw89_info(rtwdev, "R_BE_SER_L1_DBG_CNT_0=0x%08x\n", + rtw89_read32(rtwdev, R_BE_SER_L1_DBG_CNT_0)); + rtw89_info(rtwdev, "R_BE_SER_L1_DBG_CNT_1=0x%08x\n", + rtw89_read32(rtwdev, R_BE_SER_L1_DBG_CNT_1)); + rtw89_info(rtwdev, "R_BE_SER_L1_DBG_CNT_2=0x%08x\n", + rtw89_read32(rtwdev, R_BE_SER_L1_DBG_CNT_2)); + rtw89_info(rtwdev, "R_BE_SER_L1_DBG_CNT_3=0x%08x\n", + rtw89_read32(rtwdev, R_BE_SER_L1_DBG_CNT_3)); + rtw89_info(rtwdev, "R_BE_SER_L1_DBG_CNT_4=0x%08x\n", + rtw89_read32(rtwdev, R_BE_SER_L1_DBG_CNT_4)); + rtw89_info(rtwdev, "R_BE_SER_L1_DBG_CNT_5=0x%08x\n", + rtw89_read32(rtwdev, R_BE_SER_L1_DBG_CNT_5)); + rtw89_info(rtwdev, "R_BE_SER_L1_DBG_CNT_6=0x%08x\n", + rtw89_read32(rtwdev, R_BE_SER_L1_DBG_CNT_6)); + rtw89_info(rtwdev, "R_BE_SER_L1_DBG_CNT_7=0x%08x\n", + rtw89_read32(rtwdev, R_BE_SER_L1_DBG_CNT_7)); + + rtw89_mac_dump_dmac_err_status(rtwdev); + rtw89_mac_dump_cmac_err_status_be(rtwdev, RTW89_MAC_0); + rtw89_mac_dump_cmac_err_status_be(rtwdev, RTW89_MAC_1); + + rtwdev->hci.ops->dump_err_status(rtwdev); + + if (err == MAC_AX_ERR_L0_PROMOTE_TO_L1) + rtw89_mac_dump_l0_to_l1(rtwdev, err); + + rtw89_info(rtwdev, "<---\n"); +} + +static bool mac_is_txq_empty_be(struct rtw89_dev *rtwdev) +{ + struct rtw89_mac_dle_dfi_qempty qempty; + u32 val32, msk32; + u32 grpnum; + int ret; + int i; + + grpnum = rtwdev->chip->wde_qempty_acq_grpnum; + qempty.dle_type = DLE_CTRL_TYPE_WDE; + + for (i = 0; i < grpnum; i++) { + qempty.grpsel = i; + ret = rtw89_mac_dle_dfi_qempty_cfg(rtwdev, &qempty); + if (ret) { + rtw89_warn(rtwdev, + "%s: failed to dle dfi acq empty: %d\n", + __func__, ret); + return false; + } + + /* Each acq group contains 32 queues (8 macid * 4 acq), + * but here, we can simply check if all bits are set. + */ + if (qempty.qempty != MASKDWORD) + return false; + } + + qempty.grpsel = rtwdev->chip->wde_qempty_mgq_grpsel; + ret = rtw89_mac_dle_dfi_qempty_cfg(rtwdev, &qempty); + if (ret) { + rtw89_warn(rtwdev, "%s: failed to dle dfi mgq empty: %d\n", + __func__, ret); + return false; + } + + msk32 = B_CMAC0_MGQ_NORMAL_BE | B_CMAC1_MGQ_NORMAL_BE; + if ((qempty.qempty & msk32) != msk32) + return false; + + msk32 = B_BE_WDE_EMPTY_QUE_OTHERS; + val32 = rtw89_read32(rtwdev, R_BE_DLE_EMPTY0); + return (val32 & msk32) == msk32; +} + const struct rtw89_mac_gen_def rtw89_mac_gen_be = { .band1_offset = RTW89_MAC_BE_BAND_REG_OFFSET, .filter_model_addr = R_BE_FILTER_MODEL_ADDR, @@ -424,8 +1161,24 @@ const struct rtw89_mac_gen_def rtw89_mac_gen_be = { B_BE_BFMEE_HE_NDPA_EN | B_BE_BFMEE_EHT_NDPA_EN, }, + .check_mac_en = rtw89_mac_check_mac_en_be, + .hci_func_en = rtw89_mac_hci_func_en_be, + .dmac_func_pre_en = rtw89_mac_dmac_func_pre_en_be, + .dle_func_en = dle_func_en_be, + .dle_clk_en = dle_clk_en_be, .bf_assoc = rtw89_mac_bf_assoc_be, + .dle_mix_cfg = dle_mix_cfg_be, + .chk_dle_rdy = chk_dle_rdy_be, + .dle_buf_req = dle_buf_req_be, + .hfc_func_en = hfc_func_en_be, + .hfc_h2c_cfg = hfc_h2c_cfg_be, + .hfc_mix_cfg = hfc_mix_cfg_be, + .hfc_get_mix_info = hfc_get_mix_info_be, + .wde_quota_cfg = wde_quota_cfg_be, + .ple_quota_cfg = ple_quota_cfg_be, + .set_cpuio = set_cpuio_be, + .disable_cpu = rtw89_mac_disable_cpu_be, .fwdl_enable_wcpu = rtw89_mac_fwdl_enable_wcpu_be, .fwdl_get_status = fwdl_get_status_be, @@ -435,5 +1188,13 @@ const struct rtw89_mac_gen_def rtw89_mac_gen_be = { .cnv_efuse_state = rtw89_cnv_efuse_state_be, .get_txpwr_cr = rtw89_mac_get_txpwr_cr_be, + + .write_xtal_si = rtw89_mac_write_xtal_si_be, + .read_xtal_si = rtw89_mac_read_xtal_si_be, + + .dump_qta_lost = rtw89_mac_dump_qta_lost_be, + .dump_err_status = rtw89_mac_dump_err_status_be, + + .is_txq_empty = mac_is_txq_empty_be, }; EXPORT_SYMBOL(rtw89_mac_gen_be); diff --git a/drivers/net/wireless/realtek/rtw89/pci.c b/drivers/net/wireless/realtek/rtw89/pci.c index 0ca07ae63594ad..769f1ce62ebcc0 100644 --- a/drivers/net/wireless/realtek/rtw89/pci.c +++ b/drivers/net/wireless/realtek/rtw89/pci.c @@ -3826,12 +3826,22 @@ static int rtw89_pci_ops_mac_lv1_recovery(struct rtw89_dev *rtwdev, static void rtw89_pci_ops_dump_err_status(struct rtw89_dev *rtwdev) { - rtw89_info(rtwdev, "R_AX_RPQ_RXBD_IDX =0x%08x\n", - rtw89_read32(rtwdev, R_AX_RPQ_RXBD_IDX)); - rtw89_info(rtwdev, "R_AX_DBG_ERR_FLAG=0x%08x\n", - rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG)); - rtw89_info(rtwdev, "R_AX_LBC_WATCHDOG=0x%08x\n", - rtw89_read32(rtwdev, R_AX_LBC_WATCHDOG)); + if (rtwdev->chip->chip_gen == RTW89_CHIP_BE) + return; + + if (rtwdev->chip->chip_id == RTL8852C) { + rtw89_info(rtwdev, "R_AX_DBG_ERR_FLAG=0x%08x\n", + rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG_V1)); + rtw89_info(rtwdev, "R_AX_LBC_WATCHDOG=0x%08x\n", + rtw89_read32(rtwdev, R_AX_LBC_WATCHDOG_V1)); + } else { + rtw89_info(rtwdev, "R_AX_RPQ_RXBD_IDX =0x%08x\n", + rtw89_read32(rtwdev, R_AX_RPQ_RXBD_IDX)); + rtw89_info(rtwdev, "R_AX_DBG_ERR_FLAG=0x%08x\n", + rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG)); + rtw89_info(rtwdev, "R_AX_LBC_WATCHDOG=0x%08x\n", + rtw89_read32(rtwdev, R_AX_LBC_WATCHDOG)); + } } static int rtw89_pci_napi_poll(struct napi_struct *napi, int budget) diff --git a/drivers/net/wireless/realtek/rtw89/pci.h b/drivers/net/wireless/realtek/rtw89/pci.h index e2d8eef52b20e0..ca5de77fee90a6 100644 --- a/drivers/net/wireless/realtek/rtw89/pci.h +++ b/drivers/net/wireless/realtek/rtw89/pci.h @@ -812,6 +812,8 @@ #define B_AX_RXCOUNTER_MATCH_MASK GENMASK(15, 8) #define B_AX_RXTIMER_MATCH_MASK GENMASK(7, 0) +#define R_AX_DBG_ERR_FLAG_V1 0x1104 + #define R_AX_INT_MIT_RX_V1 0x1184 #define B_AX_RXMIT_RXP2_SEL_V1 BIT(19) #define B_AX_RXMIT_RXP1_SEL_V1 BIT(18) @@ -857,6 +859,8 @@ #define R_AX_PCIE_HRPWM_V1 0x30C0 #define R_AX_PCIE_CRPWM 0x30C4 +#define R_AX_LBC_WATCHDOG_V1 0x30D8 + #define R_BE_PCIE_HRPWM 0x30C0 #define R_BE_PCIE_CRPWM 0x30C4 diff --git a/drivers/net/wireless/realtek/rtw89/phy.c b/drivers/net/wireless/realtek/rtw89/phy.c index 17ccc9efed2895..bafc7b1cc10417 100644 --- a/drivers/net/wireless/realtek/rtw89/phy.c +++ b/drivers/net/wireless/realtek/rtw89/phy.c @@ -2445,6 +2445,298 @@ void (* const rtw89_phy_c2h_ra_handler[])(struct rtw89_dev *rtwdev, [RTW89_PHY_C2H_FUNC_TXSTS] = NULL, }; +static void rtw89_phy_c2h_rfk_rpt_log(struct rtw89_dev *rtwdev, + enum rtw89_phy_c2h_rfk_log_func func, + void *content, u16 len) +{ + struct rtw89_c2h_rf_txgapk_rpt_log *txgapk; + struct rtw89_c2h_rf_rxdck_rpt_log *rxdck; + struct rtw89_c2h_rf_dack_rpt_log *dack; + struct rtw89_c2h_rf_dpk_rpt_log *dpk; + + switch (func) { + case RTW89_PHY_C2H_RFK_LOG_FUNC_DPK: + if (len != sizeof(*dpk)) + goto out; + + dpk = content; + rtw89_debug(rtwdev, RTW89_DBG_RFK, + "DPK ver:%d idx:%2ph band:%2ph bw:%2ph ch:%2ph path:%2ph\n", + dpk->ver, dpk->idx, dpk->band, dpk->bw, dpk->ch, dpk->path_ok); + rtw89_debug(rtwdev, RTW89_DBG_RFK, + "DPK txagc:%2ph ther:%2ph gs:%2ph dc_i:%4ph dc_q:%4ph\n", + dpk->txagc, dpk->ther, dpk->gs, dpk->dc_i, dpk->dc_q); + rtw89_debug(rtwdev, RTW89_DBG_RFK, + "DPK corr_v:%2ph corr_i:%2ph to:%2ph ov:%2ph\n", + dpk->corr_val, dpk->corr_idx, dpk->is_timeout, dpk->rxbb_ov); + return; + case RTW89_PHY_C2H_RFK_LOG_FUNC_DACK: + if (len != sizeof(*dack)) + goto out; + + dack = content; + + rtw89_debug(rtwdev, RTW89_DBG_RFK, "[DACK]ver=0x%x 0x%x\n", + dack->fwdack_ver, dack->fwdack_rpt_ver); + rtw89_debug(rtwdev, RTW89_DBG_RFK, "[DACK]S0 CDACK ic = [0x%x, 0x%x]\n", + dack->cdack_d[0][0][0], dack->cdack_d[0][0][1]); + rtw89_debug(rtwdev, RTW89_DBG_RFK, "[DACK]S0 CDACK qc = [0x%x, 0x%x]\n", + dack->cdack_d[0][1][0], dack->cdack_d[0][1][1]); + rtw89_debug(rtwdev, RTW89_DBG_RFK, "[DACK]S1 CDACK ic = [0x%x, 0x%x]\n", + dack->cdack_d[1][0][0], dack->cdack_d[1][0][1]); + rtw89_debug(rtwdev, RTW89_DBG_RFK, "[DACK]S1 CDACK qc = [0x%x, 0x%x]\n", + dack->cdack_d[1][1][0], dack->cdack_d[1][1][1]); + + rtw89_debug(rtwdev, RTW89_DBG_RFK, "[DACK]S0 ADC_DCK ic = [0x%x, 0x%x]\n", + dack->addck2_d[0][0][0], dack->addck2_d[0][0][1]); + rtw89_debug(rtwdev, RTW89_DBG_RFK, "[DACK]S0 ADC_DCK qc = [0x%x, 0x%x]\n", + dack->addck2_d[0][1][0], dack->addck2_d[0][1][1]); + rtw89_debug(rtwdev, RTW89_DBG_RFK, "[DACK]S1 ADC_DCK ic = [0x%x, 0x%x]\n", + dack->addck2_d[1][0][0], dack->addck2_d[1][0][1]); + rtw89_debug(rtwdev, RTW89_DBG_RFK, "[DACK]S1 ADC_DCK qc = [0x%x, 0x%x]\n", + dack->addck2_d[1][1][0], dack->addck2_d[1][1][1]); + + rtw89_debug(rtwdev, RTW89_DBG_RFK, "[DACK]S0 ADC_GAINK ic = 0x%x, qc = 0x%x\n", + dack->adgaink_d[0][0], dack->adgaink_d[0][1]); + rtw89_debug(rtwdev, RTW89_DBG_RFK, "[DACK]S1 ADC_GAINK ic = 0x%x, qc = 0x%x\n", + dack->adgaink_d[1][0], dack->adgaink_d[1][1]); + + rtw89_debug(rtwdev, RTW89_DBG_RFK, "[DACK]S0 DAC_DCK ic = 0x%x, qc = 0x%x\n", + dack->dadck_d[0][0], dack->dadck_d[0][1]); + rtw89_debug(rtwdev, RTW89_DBG_RFK, "[DACK]S1 DAC_DCK ic = 0x%x, qc = 0x%x\n", + dack->dadck_d[1][0], dack->dadck_d[1][1]); + + rtw89_debug(rtwdev, RTW89_DBG_RFK, "[DACK]S0 biask iqc = 0x%x\n", + dack->biask_d[0][0]); + rtw89_debug(rtwdev, RTW89_DBG_RFK, "[DACK]S1 biask iqc = 0x%x\n", + dack->biask_d[1][0]); + + rtw89_debug(rtwdev, RTW89_DBG_RFK, "[DACK]S0 MSBK ic: %*ph\n", + (int)sizeof(dack->msbk_d[0][0]), dack->msbk_d[0][0]); + rtw89_debug(rtwdev, RTW89_DBG_RFK, "[DACK]S0 MSBK qc: %*ph\n", + (int)sizeof(dack->msbk_d[0][1]), dack->msbk_d[0][1]); + rtw89_debug(rtwdev, RTW89_DBG_RFK, "[DACK]S1 MSBK ic: %*ph\n", + (int)sizeof(dack->msbk_d[1][0]), dack->msbk_d[1][0]); + rtw89_debug(rtwdev, RTW89_DBG_RFK, "[DACK]S1 MSBK qc: %*ph\n", + (int)sizeof(dack->msbk_d[1][1]), dack->msbk_d[1][1]); + return; + case RTW89_PHY_C2H_RFK_LOG_FUNC_RXDCK: + if (len != sizeof(*rxdck)) + goto out; + + rxdck = content; + rtw89_debug(rtwdev, RTW89_DBG_RFK, + "RXDCK ver:%d band:%2ph bw:%2ph ch:%2ph to:%2ph\n", + rxdck->ver, rxdck->band, rxdck->bw, rxdck->ch, + rxdck->timeout); + return; + case RTW89_PHY_C2H_RFK_LOG_FUNC_TXGAPK: + if (len != sizeof(*txgapk)) + goto out; + + txgapk = content; + rtw89_debug(rtwdev, RTW89_DBG_RFK, + "[TXGAPK]rpt r0x8010[0]=0x%x, r0x8010[1]=0x%x\n", + le32_to_cpu(txgapk->r0x8010[0]), + le32_to_cpu(txgapk->r0x8010[1])); + rtw89_debug(rtwdev, RTW89_DBG_RFK, "[TXGAPK]rpt chk_id = %d\n", + txgapk->chk_id); + rtw89_debug(rtwdev, RTW89_DBG_RFK, "[TXGAPK]rpt chk_cnt = %d\n", + le32_to_cpu(txgapk->chk_cnt)); + rtw89_debug(rtwdev, RTW89_DBG_RFK, "[TXGAPK]rpt ver = 0x%x\n", + txgapk->ver); + rtw89_debug(rtwdev, RTW89_DBG_RFK, "[TXGAPK]rpt rsv1 = %d\n", + txgapk->rsv1); + + rtw89_debug(rtwdev, RTW89_DBG_RFK, "[TXGAPK]rpt track_d[0] = %*ph\n", + (int)sizeof(txgapk->track_d[0]), txgapk->track_d[0]); + rtw89_debug(rtwdev, RTW89_DBG_RFK, "[TXGAPK]rpt power_d[0] = %*ph\n", + (int)sizeof(txgapk->power_d[0]), txgapk->power_d[0]); + rtw89_debug(rtwdev, RTW89_DBG_RFK, "[TXGAPK]rpt track_d[1] = %*ph\n", + (int)sizeof(txgapk->track_d[1]), txgapk->track_d[1]); + rtw89_debug(rtwdev, RTW89_DBG_RFK, "[TXGAPK]rpt power_d[1] = %*ph\n", + (int)sizeof(txgapk->power_d[1]), txgapk->power_d[1]); + return; + default: + break; + } + +out: + rtw89_debug(rtwdev, RTW89_DBG_RFK, + "unexpected RFK func %d report log with length %d\n", func, len); +} + +static bool rtw89_phy_c2h_rfk_run_log(struct rtw89_dev *rtwdev, + enum rtw89_phy_c2h_rfk_log_func func, + void *content, u16 len) +{ + struct rtw89_fw_elm_info *elm_info = &rtwdev->fw.elm_info; + const struct rtw89_c2h_rf_run_log *log = content; + const struct rtw89_fw_element_hdr *elm; + u32 fmt_idx; + u16 offset; + + if (sizeof(*log) != len) + return false; + + if (!elm_info->rfk_log_fmt) + return false; + + elm = elm_info->rfk_log_fmt->elm[func]; + fmt_idx = le32_to_cpu(log->fmt_idx); + if (!elm || fmt_idx >= elm->u.rfk_log_fmt.nr) + return false; + + offset = le16_to_cpu(elm->u.rfk_log_fmt.offset[fmt_idx]); + if (offset == 0) + return false; + + rtw89_debug(rtwdev, RTW89_DBG_RFK, &elm->u.common.contents[offset], + le32_to_cpu(log->arg[0]), le32_to_cpu(log->arg[1]), + le32_to_cpu(log->arg[2]), le32_to_cpu(log->arg[3])); + + return true; +} + +static void rtw89_phy_c2h_rfk_log(struct rtw89_dev *rtwdev, struct sk_buff *c2h, + u32 len, enum rtw89_phy_c2h_rfk_log_func func, + const char *rfk_name) +{ + struct rtw89_c2h_hdr *c2h_hdr = (struct rtw89_c2h_hdr *)c2h->data; + struct rtw89_c2h_rf_log_hdr *log_hdr; + void *log_ptr = c2h_hdr; + u16 content_len; + u16 chunk_len; + bool handled; + + if (!rtw89_debug_is_enabled(rtwdev, RTW89_DBG_RFK)) + return; + + log_ptr += sizeof(*c2h_hdr); + len -= sizeof(*c2h_hdr); + + while (len > sizeof(*log_hdr)) { + log_hdr = log_ptr; + content_len = le16_to_cpu(log_hdr->len); + chunk_len = content_len + sizeof(*log_hdr); + + if (chunk_len > len) + break; + + switch (log_hdr->type) { + case RTW89_RF_RUN_LOG: + handled = rtw89_phy_c2h_rfk_run_log(rtwdev, func, + log_hdr->content, content_len); + if (handled) + break; + + rtw89_debug(rtwdev, RTW89_DBG_RFK, "%s run: %*ph\n", + rfk_name, content_len, log_hdr->content); + break; + case RTW89_RF_RPT_LOG: + rtw89_phy_c2h_rfk_rpt_log(rtwdev, func, + log_hdr->content, content_len); + break; + default: + return; + } + + log_ptr += chunk_len; + len -= chunk_len; + } +} + +static void +rtw89_phy_c2h_rfk_log_iqk(struct rtw89_dev *rtwdev, struct sk_buff *c2h, u32 len) +{ + rtw89_phy_c2h_rfk_log(rtwdev, c2h, len, + RTW89_PHY_C2H_RFK_LOG_FUNC_IQK, "IQK"); +} + +static void +rtw89_phy_c2h_rfk_log_dpk(struct rtw89_dev *rtwdev, struct sk_buff *c2h, u32 len) +{ + rtw89_phy_c2h_rfk_log(rtwdev, c2h, len, + RTW89_PHY_C2H_RFK_LOG_FUNC_DPK, "DPK"); +} + +static void +rtw89_phy_c2h_rfk_log_dack(struct rtw89_dev *rtwdev, struct sk_buff *c2h, u32 len) +{ + rtw89_phy_c2h_rfk_log(rtwdev, c2h, len, + RTW89_PHY_C2H_RFK_LOG_FUNC_DACK, "DACK"); +} + +static void +rtw89_phy_c2h_rfk_log_rxdck(struct rtw89_dev *rtwdev, struct sk_buff *c2h, u32 len) +{ + rtw89_phy_c2h_rfk_log(rtwdev, c2h, len, + RTW89_PHY_C2H_RFK_LOG_FUNC_RXDCK, "RX_DCK"); +} + +static void +rtw89_phy_c2h_rfk_log_tssi(struct rtw89_dev *rtwdev, struct sk_buff *c2h, u32 len) +{ + rtw89_phy_c2h_rfk_log(rtwdev, c2h, len, + RTW89_PHY_C2H_RFK_LOG_FUNC_TSSI, "TSSI"); +} + +static void +rtw89_phy_c2h_rfk_log_txgapk(struct rtw89_dev *rtwdev, struct sk_buff *c2h, u32 len) +{ + rtw89_phy_c2h_rfk_log(rtwdev, c2h, len, + RTW89_PHY_C2H_RFK_LOG_FUNC_TXGAPK, "TXGAPK"); +} + +static +void (* const rtw89_phy_c2h_rfk_log_handler[])(struct rtw89_dev *rtwdev, + struct sk_buff *c2h, u32 len) = { + [RTW89_PHY_C2H_RFK_LOG_FUNC_IQK] = rtw89_phy_c2h_rfk_log_iqk, + [RTW89_PHY_C2H_RFK_LOG_FUNC_DPK] = rtw89_phy_c2h_rfk_log_dpk, + [RTW89_PHY_C2H_RFK_LOG_FUNC_DACK] = rtw89_phy_c2h_rfk_log_dack, + [RTW89_PHY_C2H_RFK_LOG_FUNC_RXDCK] = rtw89_phy_c2h_rfk_log_rxdck, + [RTW89_PHY_C2H_RFK_LOG_FUNC_TSSI] = rtw89_phy_c2h_rfk_log_tssi, + [RTW89_PHY_C2H_RFK_LOG_FUNC_TXGAPK] = rtw89_phy_c2h_rfk_log_txgapk, +}; + +static void +rtw89_phy_c2h_rfk_report_state(struct rtw89_dev *rtwdev, struct sk_buff *c2h, u32 len) +{ +} + +static +void (* const rtw89_phy_c2h_rfk_report_handler[])(struct rtw89_dev *rtwdev, + struct sk_buff *c2h, u32 len) = { + [RTW89_PHY_C2H_RFK_REPORT_FUNC_STATE] = rtw89_phy_c2h_rfk_report_state, +}; + +bool rtw89_phy_c2h_chk_atomic(struct rtw89_dev *rtwdev, u8 class, u8 func) +{ + switch (class) { + case RTW89_PHY_C2H_RFK_LOG: + switch (func) { + case RTW89_PHY_C2H_RFK_LOG_FUNC_IQK: + case RTW89_PHY_C2H_RFK_LOG_FUNC_DPK: + case RTW89_PHY_C2H_RFK_LOG_FUNC_DACK: + case RTW89_PHY_C2H_RFK_LOG_FUNC_RXDCK: + case RTW89_PHY_C2H_RFK_LOG_FUNC_TSSI: + case RTW89_PHY_C2H_RFK_LOG_FUNC_TXGAPK: + return true; + default: + return false; + } + case RTW89_PHY_C2H_RFK_REPORT: + switch (func) { + case RTW89_PHY_C2H_RFK_REPORT_FUNC_STATE: + return true; + default: + return false; + } + default: + return false; + } +} + void rtw89_phy_c2h_handle(struct rtw89_dev *rtwdev, struct sk_buff *skb, u32 len, u8 class, u8 func) { @@ -2456,6 +2748,14 @@ void rtw89_phy_c2h_handle(struct rtw89_dev *rtwdev, struct sk_buff *skb, if (func < RTW89_PHY_C2H_FUNC_RA_MAX) handler = rtw89_phy_c2h_ra_handler[func]; break; + case RTW89_PHY_C2H_RFK_LOG: + if (func < ARRAY_SIZE(rtw89_phy_c2h_rfk_log_handler)) + handler = rtw89_phy_c2h_rfk_log_handler[func]; + break; + case RTW89_PHY_C2H_RFK_REPORT: + if (func < ARRAY_SIZE(rtw89_phy_c2h_rfk_report_handler)) + handler = rtw89_phy_c2h_rfk_report_handler[func]; + break; case RTW89_PHY_C2H_CLASS_DM: if (func == RTW89_PHY_C2H_DM_FUNC_LOWRT_RTY) return; @@ -4620,6 +4920,29 @@ static void rtw89_phy_env_monitor_init(struct rtw89_dev *rtwdev) rtw89_phy_ifs_clm_setting_init(rtwdev); } +static void rtw89_phy_edcca_init(struct rtw89_dev *rtwdev) +{ + const struct rtw89_edcca_regs *edcca_regs = rtwdev->chip->edcca_regs; + struct rtw89_edcca_bak *edcca_bak = &rtwdev->hal.edcca_bak; + + memset(edcca_bak, 0, sizeof(*edcca_bak)); + + if (rtwdev->chip->chip_id == RTL8922A && rtwdev->hal.cv == CHIP_CAV) { + rtw89_phy_set_phy_regs(rtwdev, R_TXGATING, B_TXGATING_EN, 0); + rtw89_phy_set_phy_regs(rtwdev, R_CTLTOP, B_CTLTOP_VAL, 2); + rtw89_phy_set_phy_regs(rtwdev, R_CTLTOP, B_CTLTOP_ON, 1); + rtw89_phy_set_phy_regs(rtwdev, R_SPOOF_CG, B_SPOOF_CG_EN, 0); + rtw89_phy_set_phy_regs(rtwdev, R_DFS_FFT_CG, B_DFS_CG_EN, 0); + rtw89_phy_set_phy_regs(rtwdev, R_DFS_FFT_CG, B_DFS_FFT_EN, 0); + rtw89_phy_set_phy_regs(rtwdev, R_SEGSND, B_SEGSND_EN, 0); + rtw89_phy_set_phy_regs(rtwdev, R_SEGSND, B_SEGSND_EN, 1); + rtw89_phy_set_phy_regs(rtwdev, R_DFS_FFT_CG, B_DFS_FFT_EN, 1); + } + + rtw89_phy_write32_mask(rtwdev, edcca_regs->tx_collision_t2r_st, + edcca_regs->tx_collision_t2r_st_mask, 0x29); +} + void rtw89_phy_dm_init(struct rtw89_dev *rtwdev) { rtw89_phy_stat_init(rtwdev); @@ -4630,6 +4953,7 @@ void rtw89_phy_dm_init(struct rtw89_dev *rtwdev) rtw89_physts_parsing_init(rtwdev); rtw89_phy_dig_init(rtwdev); rtw89_phy_cfo_init(rtwdev); + rtw89_phy_edcca_init(rtwdev); rtw89_phy_ul_tb_info_init(rtwdev); rtw89_phy_antdiv_init(rtwdev); rtw89_chip_rfe_gpio(rtwdev); @@ -4892,23 +5216,188 @@ void rtw89_decode_chan_idx(struct rtw89_dev *rtwdev, u8 chan_idx, } EXPORT_SYMBOL(rtw89_decode_chan_idx); -#define EDCCA_DEFAULT 249 void rtw89_phy_config_edcca(struct rtw89_dev *rtwdev, bool scan) { - u32 reg = rtwdev->chip->edcca_lvl_reg; - struct rtw89_hal *hal = &rtwdev->hal; - u32 val; + const struct rtw89_edcca_regs *edcca_regs = rtwdev->chip->edcca_regs; + struct rtw89_edcca_bak *edcca_bak = &rtwdev->hal.edcca_bak; if (scan) { - hal->edcca_bak = rtw89_phy_read32(rtwdev, reg); - val = hal->edcca_bak; - u32p_replace_bits(&val, EDCCA_DEFAULT, B_SEG0R_EDCCA_LVL_A_MSK); - u32p_replace_bits(&val, EDCCA_DEFAULT, B_SEG0R_EDCCA_LVL_P_MSK); - u32p_replace_bits(&val, EDCCA_DEFAULT, B_SEG0R_PPDU_LVL_MSK); - rtw89_phy_write32(rtwdev, reg, val); + edcca_bak->a = + rtw89_phy_read32_mask(rtwdev, edcca_regs->edcca_level, + edcca_regs->edcca_mask); + edcca_bak->p = + rtw89_phy_read32_mask(rtwdev, edcca_regs->edcca_level, + edcca_regs->edcca_p_mask); + edcca_bak->ppdu = + rtw89_phy_read32_mask(rtwdev, edcca_regs->ppdu_level, + edcca_regs->ppdu_mask); + + rtw89_phy_write32_mask(rtwdev, edcca_regs->edcca_level, + edcca_regs->edcca_mask, EDCCA_MAX); + rtw89_phy_write32_mask(rtwdev, edcca_regs->edcca_level, + edcca_regs->edcca_p_mask, EDCCA_MAX); + rtw89_phy_write32_mask(rtwdev, edcca_regs->ppdu_level, + edcca_regs->ppdu_mask, EDCCA_MAX); + } else { + rtw89_phy_write32_mask(rtwdev, edcca_regs->edcca_level, + edcca_regs->edcca_mask, + edcca_bak->a); + rtw89_phy_write32_mask(rtwdev, edcca_regs->edcca_level, + edcca_regs->edcca_p_mask, + edcca_bak->p); + rtw89_phy_write32_mask(rtwdev, edcca_regs->ppdu_level, + edcca_regs->ppdu_mask, + edcca_bak->ppdu); + } +} + +static void rtw89_phy_edcca_log(struct rtw89_dev *rtwdev) +{ + const struct rtw89_edcca_regs *edcca_regs = rtwdev->chip->edcca_regs; + bool flag_fb, flag_p20, flag_s20, flag_s40, flag_s80; + s8 pwdb_fb, pwdb_p20, pwdb_s20, pwdb_s40, pwdb_s80; + u8 path, per20_bitmap; + u8 pwdb[8]; + u32 tmp; + + if (!rtw89_debug_is_enabled(rtwdev, RTW89_DBG_EDCCA)) + return; + + if (rtwdev->chip->chip_id == RTL8922A) + rtw89_phy_write32_mask(rtwdev, edcca_regs->rpt_sel_be, + edcca_regs->rpt_sel_be_mask, 0); + + rtw89_phy_write32_mask(rtwdev, edcca_regs->rpt_sel, + edcca_regs->rpt_sel_mask, 0); + tmp = rtw89_phy_read32(rtwdev, edcca_regs->rpt_b); + path = u32_get_bits(tmp, B_EDCCA_RPT_B_PATH_MASK); + flag_s80 = u32_get_bits(tmp, B_EDCCA_RPT_B_S80); + flag_s40 = u32_get_bits(tmp, B_EDCCA_RPT_B_S40); + flag_s20 = u32_get_bits(tmp, B_EDCCA_RPT_B_S20); + flag_p20 = u32_get_bits(tmp, B_EDCCA_RPT_B_P20); + flag_fb = u32_get_bits(tmp, B_EDCCA_RPT_B_FB); + pwdb_s20 = u32_get_bits(tmp, MASKBYTE1); + pwdb_p20 = u32_get_bits(tmp, MASKBYTE2); + pwdb_fb = u32_get_bits(tmp, MASKBYTE3); + + rtw89_phy_write32_mask(rtwdev, edcca_regs->rpt_sel, + edcca_regs->rpt_sel_mask, 4); + tmp = rtw89_phy_read32(rtwdev, edcca_regs->rpt_b); + pwdb_s80 = u32_get_bits(tmp, MASKBYTE1); + pwdb_s40 = u32_get_bits(tmp, MASKBYTE2); + + per20_bitmap = rtw89_phy_read32_mask(rtwdev, edcca_regs->rpt_a, + MASKBYTE0); + + if (rtwdev->chip->chip_id == RTL8922A) { + rtw89_phy_write32_mask(rtwdev, edcca_regs->rpt_sel_be, + edcca_regs->rpt_sel_be_mask, 4); + tmp = rtw89_phy_read32(rtwdev, edcca_regs->rpt_b); + pwdb[0] = u32_get_bits(tmp, MASKBYTE3); + pwdb[1] = u32_get_bits(tmp, MASKBYTE2); + pwdb[2] = u32_get_bits(tmp, MASKBYTE1); + pwdb[3] = u32_get_bits(tmp, MASKBYTE0); + + rtw89_phy_write32_mask(rtwdev, edcca_regs->rpt_sel_be, + edcca_regs->rpt_sel_be_mask, 5); + tmp = rtw89_phy_read32(rtwdev, edcca_regs->rpt_b); + pwdb[4] = u32_get_bits(tmp, MASKBYTE3); + pwdb[5] = u32_get_bits(tmp, MASKBYTE2); + pwdb[6] = u32_get_bits(tmp, MASKBYTE1); + pwdb[7] = u32_get_bits(tmp, MASKBYTE0); } else { - rtw89_phy_write32(rtwdev, reg, hal->edcca_bak); + rtw89_phy_write32_mask(rtwdev, edcca_regs->rpt_sel, + edcca_regs->rpt_sel_mask, 0); + tmp = rtw89_phy_read32(rtwdev, edcca_regs->rpt_a); + pwdb[0] = u32_get_bits(tmp, MASKBYTE3); + pwdb[1] = u32_get_bits(tmp, MASKBYTE2); + + rtw89_phy_write32_mask(rtwdev, edcca_regs->rpt_sel, + edcca_regs->rpt_sel_mask, 1); + tmp = rtw89_phy_read32(rtwdev, edcca_regs->rpt_a); + pwdb[2] = u32_get_bits(tmp, MASKBYTE3); + pwdb[3] = u32_get_bits(tmp, MASKBYTE2); + + rtw89_phy_write32_mask(rtwdev, edcca_regs->rpt_sel, + edcca_regs->rpt_sel_mask, 2); + tmp = rtw89_phy_read32(rtwdev, edcca_regs->rpt_a); + pwdb[4] = u32_get_bits(tmp, MASKBYTE3); + pwdb[5] = u32_get_bits(tmp, MASKBYTE2); + + rtw89_phy_write32_mask(rtwdev, edcca_regs->rpt_sel, + edcca_regs->rpt_sel_mask, 3); + tmp = rtw89_phy_read32(rtwdev, edcca_regs->rpt_a); + pwdb[6] = u32_get_bits(tmp, MASKBYTE3); + pwdb[7] = u32_get_bits(tmp, MASKBYTE2); } + + rtw89_debug(rtwdev, RTW89_DBG_EDCCA, + "[EDCCA]: edcca_bitmap = %04x\n", per20_bitmap); + + rtw89_debug(rtwdev, RTW89_DBG_EDCCA, + "[EDCCA]: pwdb per20{0,1,2,3,4,5,6,7} = {%d,%d,%d,%d,%d,%d,%d,%d}(dBm)\n", + pwdb[0], pwdb[1], pwdb[2], pwdb[3], pwdb[4], pwdb[5], + pwdb[6], pwdb[7]); + + rtw89_debug(rtwdev, RTW89_DBG_EDCCA, + "[EDCCA]: path=%d, flag {FB,p20,s20,s40,s80} = {%d,%d,%d,%d,%d}\n", + path, flag_fb, flag_p20, flag_s20, flag_s40, flag_s80); + + rtw89_debug(rtwdev, RTW89_DBG_EDCCA, + "[EDCCA]: pwdb {FB,p20,s20,s40,s80} = {%d,%d,%d,%d,%d}(dBm)\n", + pwdb_fb, pwdb_p20, pwdb_s20, pwdb_s40, pwdb_s80); +} + +static u8 rtw89_phy_edcca_get_thre_by_rssi(struct rtw89_dev *rtwdev) +{ + struct rtw89_phy_ch_info *ch_info = &rtwdev->ch_info; + bool is_linked = rtwdev->total_sta_assoc > 0; + u8 rssi_min = ch_info->rssi_min >> 1; + u8 edcca_thre; + + if (!is_linked) { + edcca_thre = EDCCA_MAX; + } else { + edcca_thre = rssi_min - RSSI_UNIT_CONVER + EDCCA_UNIT_CONVER - + EDCCA_TH_REF; + edcca_thre = max_t(u8, edcca_thre, EDCCA_TH_L2H_LB); + } + + return edcca_thre; +} + +void rtw89_phy_edcca_thre_calc(struct rtw89_dev *rtwdev) +{ + const struct rtw89_edcca_regs *edcca_regs = rtwdev->chip->edcca_regs; + struct rtw89_edcca_bak *edcca_bak = &rtwdev->hal.edcca_bak; + u8 th; + + th = rtw89_phy_edcca_get_thre_by_rssi(rtwdev); + if (th == edcca_bak->th_old) + return; + + edcca_bak->th_old = th; + + rtw89_debug(rtwdev, RTW89_DBG_EDCCA, + "[EDCCA]: Normal Mode, EDCCA_th = %d\n", th); + + rtw89_phy_write32_mask(rtwdev, edcca_regs->edcca_level, + edcca_regs->edcca_mask, th); + rtw89_phy_write32_mask(rtwdev, edcca_regs->edcca_level, + edcca_regs->edcca_p_mask, th); + rtw89_phy_write32_mask(rtwdev, edcca_regs->ppdu_level, + edcca_regs->ppdu_mask, th); +} + +void rtw89_phy_edcca_track(struct rtw89_dev *rtwdev) +{ + struct rtw89_hal *hal = &rtwdev->hal; + + if (hal->disabled_dm_bitmap & BIT(RTW89_DM_DYNAMIC_EDCCA)) + return; + + rtw89_phy_edcca_thre_calc(rtwdev); + rtw89_phy_edcca_log(rtwdev); } static const struct rtw89_ccx_regs rtw89_ccx_regs_ax = { diff --git a/drivers/net/wireless/realtek/rtw89/phy.h b/drivers/net/wireless/realtek/rtw89/phy.h index 2d9cf5c02b92cd..3e379077c6cadc 100644 --- a/drivers/net/wireless/realtek/rtw89/phy.h +++ b/drivers/net/wireless/realtek/rtw89/phy.h @@ -122,6 +122,13 @@ #define PHYSTS_RSVD BIT(RTW89_RX_TYPE_RSVD) #define PPDU_FILTER_BITMAP (PHYSTS_MGNT | PHYSTS_DATA) +#define EDCCA_MAX 249 +#define EDCCA_TH_L2H_LB 66 +#define EDCCA_TH_REF 3 +#define EDCCA_HL_DIFF_NORMAL 8 +#define RSSI_UNIT_CONVER 110 +#define EDCCA_UNIT_CONVER 128 + enum rtw89_phy_c2h_ra_func { RTW89_PHY_C2H_FUNC_STS_RPT, RTW89_PHY_C2H_FUNC_MU_GPTBL_RPT, @@ -129,6 +136,21 @@ enum rtw89_phy_c2h_ra_func { RTW89_PHY_C2H_FUNC_RA_MAX, }; +enum rtw89_phy_c2h_rfk_log_func { + RTW89_PHY_C2H_RFK_LOG_FUNC_IQK = 0, + RTW89_PHY_C2H_RFK_LOG_FUNC_DPK = 1, + RTW89_PHY_C2H_RFK_LOG_FUNC_DACK = 2, + RTW89_PHY_C2H_RFK_LOG_FUNC_RXDCK = 3, + RTW89_PHY_C2H_RFK_LOG_FUNC_TSSI = 4, + RTW89_PHY_C2H_RFK_LOG_FUNC_TXGAPK = 5, + + RTW89_PHY_C2H_RFK_LOG_FUNC_NUM, +}; + +enum rtw89_phy_c2h_rfk_report_func { + RTW89_PHY_C2H_RFK_REPORT_FUNC_STATE = 0, +}; + enum rtw89_phy_c2h_dm_func { RTW89_PHY_C2H_DM_FUNC_FW_TEST, RTW89_PHY_C2H_DM_FUNC_FW_TRIG_TX_RPT, @@ -142,6 +164,8 @@ enum rtw89_phy_c2h_class { RTW89_PHY_C2H_CLASS_RUA, RTW89_PHY_C2H_CLASS_RA, RTW89_PHY_C2H_CLASS_DM, + RTW89_PHY_C2H_RFK_LOG = 0x8, + RTW89_PHY_C2H_RFK_REPORT = 0x9, RTW89_PHY_C2H_CLASS_BTC_MIN = 0x10, RTW89_PHY_C2H_CLASS_BTC_MAX = 0x17, RTW89_PHY_C2H_CLASS_MAX, @@ -284,8 +308,6 @@ struct rtw89_txpwr_byrate_cfg { u32 data; }; -#define DELTA_SWINGIDX_SIZE 30 - struct rtw89_txpwr_track_cfg { const s8 (*delta_swingidx_6gb_n)[DELTA_SWINGIDX_SIZE]; const s8 (*delta_swingidx_6gb_p)[DELTA_SWINGIDX_SIZE]; @@ -478,6 +500,10 @@ struct rtw89_txpwr_limit_ru_be { s8 ru106_26[RTW89_RU_SEC_NUM_BE]; }; +struct rtw89_phy_rfk_log_fmt { + const struct rtw89_fw_element_hdr *elm[RTW89_PHY_C2H_RFK_LOG_FUNC_NUM]; +}; + struct rtw89_phy_gen_def { u32 cr_base; const struct rtw89_ccx_regs *ccx; @@ -780,6 +806,7 @@ void rtw89_phy_ra_updata_sta(struct rtw89_dev *rtwdev, struct ieee80211_sta *sta void rtw89_phy_rate_pattern_vif(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif, const struct cfg80211_bitrate_mask *mask); +bool rtw89_phy_c2h_chk_atomic(struct rtw89_dev *rtwdev, u8 class, u8 func); void rtw89_phy_c2h_handle(struct rtw89_dev *rtwdev, struct sk_buff *skb, u32 len, u8 class, u8 func); void rtw89_phy_cfo_track(struct rtw89_dev *rtwdev); @@ -807,5 +834,7 @@ u8 rtw89_encode_chan_idx(struct rtw89_dev *rtwdev, u8 central_ch, u8 band); void rtw89_decode_chan_idx(struct rtw89_dev *rtwdev, u8 chan_idx, u8 *ch, enum nl80211_band *band); void rtw89_phy_config_edcca(struct rtw89_dev *rtwdev, bool scan); +void rtw89_phy_edcca_track(struct rtw89_dev *rtwdev); +void rtw89_phy_edcca_thre_calc(struct rtw89_dev *rtwdev); #endif diff --git a/drivers/net/wireless/realtek/rtw89/ps.h b/drivers/net/wireless/realtek/rtw89/ps.h index aff0fba71cb0b7..54486e4550b61e 100644 --- a/drivers/net/wireless/realtek/rtw89/ps.h +++ b/drivers/net/wireless/realtek/rtw89/ps.h @@ -33,6 +33,10 @@ static inline void rtw89_enter_ips_by_hwflags(struct rtw89_dev *rtwdev) { struct ieee80211_hw *hw = rtwdev->hw; + /* prevent entering IPS after ROC, but it is scanning */ + if (rtwdev->scanning) + return; + if (hw->conf.flags & IEEE80211_CONF_IDLE) rtw89_enter_ips(rtwdev); } diff --git a/drivers/net/wireless/realtek/rtw89/reg.h b/drivers/net/wireless/realtek/rtw89/reg.h index 7a9ae6cd86e557..65d8a0f36700a4 100644 --- a/drivers/net/wireless/realtek/rtw89/reg.h +++ b/drivers/net/wireless/realtek/rtw89/reg.h @@ -1457,6 +1457,8 @@ #define B_AX_PLE_Q6_MAX_SIZE_MASK GENMASK(27, 16) #define B_AX_PLE_Q6_MIN_SIZE_MASK GENMASK(11, 0) #define R_AX_PLE_QTA7_CFG 0x905C +#define B_AX_PLE_Q7_MAX_SIZE_MASK GENMASK(27, 16) +#define B_AX_PLE_Q7_MIN_SIZE_MASK GENMASK(11, 0) #define R_AX_PLE_QTA8_CFG 0x9060 #define R_AX_PLE_QTA9_CFG 0x9064 #define R_AX_PLE_QTA10_CFG 0x9068 @@ -2376,6 +2378,14 @@ #define R_AX_TSFTR_HIGH_P4 0xC53C #define B_AX_TSFTR_HIGH_MASK GENMASK(31, 0) +#define R_AX_BCN_DROP_ALL0 0xC560 +#define R_AX_BCN_DROP_ALL0_C1 0xE560 +#define B_AX_BCN_DROP_ALL_P4 BIT(4) +#define B_AX_BCN_DROP_ALL_P3 BIT(3) +#define B_AX_BCN_DROP_ALL_P2 BIT(2) +#define B_AX_BCN_DROP_ALL_P1 BIT(1) +#define B_AX_BCN_DROP_ALL_P0 BIT(0) + #define R_AX_MBSSID_CTRL 0xC568 #define R_AX_MBSSID_CTRL_C1 0xE568 #define B_AX_P0MB_ALL_MASK GENMASK(23, 1) @@ -2555,11 +2565,20 @@ #define R_AX_PTCL_DBG_INFO 0xC6F0 #define R_AX_PTCL_DBG_INFO_C1 0xE6F0 +#define B_AX_PTCL_DBG_INFO_MASK_BY_PORT(port) \ +({\ + typeof(port) _port = (port); \ + GENMASK((_port) * 2 + 1, (_port) * 2); \ +}) + #define B_AX_PTCL_DBG_INFO_MASK GENMASK(31, 0) #define R_AX_PTCL_DBG 0xC6F4 #define R_AX_PTCL_DBG_C1 0xE6F4 #define B_AX_PTCL_DBG_EN BIT(8) #define B_AX_PTCL_DBG_SEL_MASK GENMASK(7, 0) +#define AX_PTCL_DBG_BCNQ_NUM0 8 +#define AX_PTCL_DBG_BCNQ_NUM1 9 + #define R_AX_DLE_CTRL 0xC800 #define R_AX_DLE_CTRL_C1 0xE800 @@ -3723,6 +3742,77 @@ #define B_BE_DIS_CLK_REG1_GATE BIT(1) #define B_BE_DIS_CLK_REG0_GATE BIT(0) +#define R_BE_ANAPAR_POW_MAC 0x0016 +#define B_BE_POW_PC_LDO_PORT1 BIT(3) +#define B_BE_POW_PC_LDO_PORT0 BIT(2) +#define B_BE_POW_PLL_V1 BIT(1) +#define B_BE_POW_POWER_CUT_POW_LDO BIT(0) + +#define R_BE_SYS_ADIE_PAD_PWR_CTRL 0x0018 +#define B_BE_SYM_PADPDN_WL_RFC1_1P3 BIT(6) +#define B_BE_SYM_PADPDN_WL_RFC0_1P3 BIT(5) + +#define R_BE_AFE_LDO_CTRL 0x0020 +#define B_BE_FORCE_MACBBBT_PWR_ON BIT(31) +#define B_BE_R_SYM_WLPOFF_P4_PC_EN BIT(28) +#define B_BE_R_SYM_WLPOFF_P3_PC_EN BIT(27) +#define B_BE_R_SYM_WLPOFF_P2_PC_EN BIT(26) +#define B_BE_R_SYM_WLPOFF_P1_PC_EN BIT(25) +#define B_BE_R_SYM_WLPOFF_PC_EN BIT(24) +#define B_BE_AON_OFF_PC_EN BIT(23) +#define B_BE_R_SYM_WLPON_P3_PC_EN BIT(21) +#define B_BE_R_SYM_WLPON_P2_PC_EN BIT(20) +#define B_BE_R_SYM_WLPON_P1_PC_EN BIT(19) +#define B_BE_R_SYM_WLPON_PC_EN BIT(18) +#define B_BE_R_SYM_WLBBPON1_P1_PC_EN BIT(15) +#define B_BE_R_SYM_WLBBPON1_PC_EN BIT(14) +#define B_BE_R_SYM_WLBBPON_P1_PC_EN BIT(13) +#define B_BE_R_SYM_WLBBPON_PC_EN BIT(12) +#define B_BE_R_SYM_DIS_WPHYBBOFF_PC BIT(10) +#define B_BE_R_SYM_WLBBOFF1_P4_PC_EN BIT(9) +#define B_BE_R_SYM_WLBBOFF1_P3_PC_EN BIT(8) +#define B_BE_R_SYM_WLBBOFF1_P2_PC_EN BIT(7) +#define B_BE_R_SYM_WLBBOFF1_P1_PC_EN BIT(6) +#define B_BE_R_SYM_WLBBOFF1_PC_EN BIT(5) +#define B_BE_R_SYM_WLBBOFF_P4_PC_EN BIT(4) +#define B_BE_R_SYM_WLBBOFF_P3_PC_EN BIT(3) +#define B_BE_R_SYM_WLBBOFF_P2_PC_EN BIT(2) +#define B_BE_R_SYM_WLBBOFF_P1_PC_EN BIT(1) +#define B_BE_R_SYM_WLBBOFF_PC_EN BIT(0) + +#define R_BE_AFE_CTRL1 0x0024 +#define B_BE_R_SYM_WLCMAC0_P4_PC_EN BIT(28) +#define B_BE_R_SYM_WLCMAC0_P3_PC_EN BIT(27) +#define B_BE_R_SYM_WLCMAC0_P2_PC_EN BIT(26) +#define B_BE_R_SYM_WLCMAC0_P1_PC_EN BIT(25) +#define B_BE_R_SYM_WLCMAC0_PC_EN BIT(24) +#define B_BE_DATAMEM_PC3_EN BIT(23) +#define B_BE_DATAMEM_PC2_EN BIT(22) +#define B_BE_DATAMEM_PC1_EN BIT(21) +#define B_BE_DATAMEM_PC_EN BIT(20) +#define B_BE_DMEM7_PC_EN BIT(19) +#define B_BE_DMEM6_PC_EN BIT(18) +#define B_BE_DMEM5_PC_EN BIT(17) +#define B_BE_DMEM4_PC_EN BIT(16) +#define B_BE_DMEM3_PC_EN BIT(15) +#define B_BE_DMEM2_PC_EN BIT(14) +#define B_BE_DMEM1_PC_EN BIT(13) +#define B_BE_IMEM4_PC_EN BIT(12) +#define B_BE_IMEM3_PC_EN BIT(11) +#define B_BE_IMEM2_PC_EN BIT(10) +#define B_BE_IMEM1_PC_EN BIT(9) +#define B_BE_IMEM0_PC_EN BIT(8) +#define B_BE_R_SYM_WLCMAC1_P4_PC_EN BIT(4) +#define B_BE_R_SYM_WLCMAC1_P3_PC_EN BIT(3) +#define B_BE_R_SYM_WLCMAC1_P2_PC_EN BIT(2) +#define B_BE_R_SYM_WLCMAC1_P1_PC_EN BIT(1) +#define B_BE_R_SYM_WLCMAC1_PC_EN BIT(0) +#define B_BE_AFE_CTRL1_SET (B_BE_R_SYM_WLCMAC1_PC_EN | \ + B_BE_R_SYM_WLCMAC1_P1_PC_EN | \ + B_BE_R_SYM_WLCMAC1_P2_PC_EN | \ + B_BE_R_SYM_WLCMAC1_P3_PC_EN | \ + B_BE_R_SYM_WLCMAC1_P4_PC_EN) + #define R_BE_EFUSE_CTRL 0x0030 #define B_BE_EF_MODE_SEL_MASK GENMASK(31, 30) #define B_BE_EF_RDY BIT(29) @@ -3810,6 +3900,52 @@ #define B_BE_PCI_CKRDY_OPT BIT(1) #define B_BE_PCI_VAUX_EN BIT(0) +#define R_BE_SYS_ISO_CTRL_EXTEND 0x0080 +#define B_BE_R_SYM_ISO_DMEM62PP BIT(29) +#define B_BE_R_SYM_ISO_DMEM52PP BIT(28) +#define B_BE_R_SYM_ISO_DMEM42PP BIT(27) +#define B_BE_R_SYM_ISO_DMEM32PP BIT(26) +#define B_BE_R_SYM_ISO_DMEM22PP BIT(25) +#define B_BE_R_SYM_ISO_DMEM12PP BIT(24) +#define B_BE_R_SYM_ISO_IMEM42PP BIT(22) +#define B_BE_R_SYM_ISO_IMEM32PP BIT(21) +#define B_BE_R_SYM_ISO_IMEM22PP BIT(20) +#define B_BE_R_SYM_ISO_IMEM12PP BIT(19) +#define B_BE_R_SYM_ISO_IMEM02PP BIT(18) +#define B_BE_R_SYM_ISO_AON_OFF2PP BIT(15) +#define B_BE_R_SYM_PWC_HCILA BIT(13) +#define B_BE_R_SYM_PWC_PD12V BIT(12) +#define B_BE_R_SYM_PWC_UD12V BIT(11) +#define B_BE_R_SYM_PWC_BTBRG BIT(10) +#define B_BE_R_SYM_LDOBTSDIO_EN BIT(9) +#define B_BE_R_SYM_LDOSPDIO_EN BIT(8) +#define B_BE_R_SYM_ISO_HCILA BIT(4) +#define B_BE_R_SYM_ISO_BTBRG2PP BIT(2) +#define B_BE_R_SYM_ISO_BTSDIO2PP BIT(1) +#define B_BE_R_SYM_ISO_SPDIO2PP BIT(0) + +#define R_BE_FEN_RST_ENABLE 0x0084 +#define B_BE_R_SYM_FEN_WLMACOFF BIT(31) +#define B_BE_R_SYM_ISO_WA12PP BIT(28) +#define B_BE_R_SYM_ISO_CMAC12PP BIT(25) +#define B_BE_R_SYM_ISO_CMAC02PP BIT(24) +#define B_BE_R_SYM_ISO_ADDA_P32PP BIT(23) +#define B_BE_R_SYM_ISO_ADDA_P22PP BIT(22) +#define B_BE_R_SYM_ISO_ADDA_P12PP BIT(21) +#define B_BE_R_SYM_ISO_ADDA_P02PP BIT(20) +#define B_BE_CMAC1_FEN BIT(17) +#define B_BE_CMAC0_FEN BIT(16) +#define B_BE_SYM_ISO_BBPON12PP BIT(13) +#define B_BE_SYM_ISO_BB12PP BIT(12) +#define B_BE_BOOT_RDY1 BIT(10) +#define B_BE_FEN_BB1_IP_RSTN BIT(9) +#define B_BE_FEN_BB1PLAT_RSTB BIT(8) +#define B_BE_SYM_ISO_BBPON02PP BIT(5) +#define B_BE_SYM_ISO_BB02PP BIT(4) +#define B_BE_BOOT_RDY0 BIT(2) +#define B_BE_FEN_BB_IP_RSTN BIT(1) +#define B_BE_FEN_BBPLAT_RSTB BIT(0) + #define R_BE_PLATFORM_ENABLE 0x0088 #define B_BE_HOLD_AFTER_RESET BIT(11) #define B_BE_SYM_WLPLT_MEM_MUX_EN BIT(10) @@ -3823,6 +3959,51 @@ #define B_BE_WCPU_EN BIT(1) #define B_BE_PLATFORM_EN BIT(0) +#define R_BE_WLLPS_CTRL 0x0090 +#define B_BE_LPSOP_BBMEMDS BIT(30) +#define B_BE_LPSOP_BBOFF BIT(29) +#define B_BE_LPSOP_MACOFF BIT(28) +#define B_BE_LPSOP_OFF_CAPC_EN BIT(27) +#define B_BE_LPSOP_MEM_DS BIT(26) +#define B_BE_LPSOP_XTALM_LPS BIT(23) +#define B_BE_LPSOP_XTAL BIT(22) +#define B_BE_LPSOP_ACLK_DIV_2 BIT(21) +#define B_BE_LPSOP_ACLK_SEL BIT(20) +#define B_BE_LPSOP_ASWRM BIT(17) +#define B_BE_LPSOP_ASWR BIT(16) +#define B_BE_LPSOP_DSWR_ADJ_MASK GENMASK(15, 12) +#define B_BE_LPSOP_DSWRSD BIT(10) +#define B_BE_LPSOP_DSWRM BIT(9) +#define B_BE_LPSOP_DSWR BIT(8) +#define B_BE_LPSOP_OLD_ADJ_MASK GENMASK(7, 4) +#define B_BE_FORCE_LEAVE_LPS BIT(3) +#define B_BE_LPSOP_OLDSD BIT(2) +#define B_BE_DIS_WLBT_LPSEN_LOPC BIT(1) +#define B_BE_WL_LPS_EN BIT(0) + +#define R_BE_WLRESUME_CTRL 0x0094 +#define B_BE_LPSROP_DMEM5_RSU_EN BIT(31) +#define B_BE_LPSROP_DMEM4_RSU_EN BIT(30) +#define B_BE_LPSROP_DMEM3_RSU_EN BIT(29) +#define B_BE_LPSROP_DMEM2_RSU_EN BIT(28) +#define B_BE_LPSROP_DMEM1_RSU_EN BIT(27) +#define B_BE_LPSROP_DMEM0_RSU_EN BIT(26) +#define B_BE_LPSROP_IMEM5_RSU_EN BIT(25) +#define B_BE_LPSROP_IMEM4_RSU_EN BIT(24) +#define B_BE_LPSROP_IMEM3_RSU_EN BIT(23) +#define B_BE_LPSROP_IMEM2_RSU_EN BIT(22) +#define B_BE_LPSROP_IMEM1_RSU_EN BIT(21) +#define B_BE_LPSROP_IMEM0_RSU_EN BIT(20) +#define B_BE_LPSROP_BB1_W_BB0 BIT(14) +#define B_BE_LPSROP_CMAC1 BIT(13) +#define B_BE_LPSROP_CMAC0 BIT(12) +#define B_BE_LPSROP_XTALM BIT(11) +#define B_BE_LPSROP_PLLM BIT(10) +#define B_BE_LPSROP_HIOE BIT(9) +#define B_BE_LPSROP_CPU BIT(8) +#define B_BE_LPSROP_LOWPWRPLL BIT(7) +#define B_BE_LPSROP_DSWRSD_SEL_MASK GENMASK(5, 4) + #define R_BE_EFUSE_CTRL_2_V1 0x00A4 #define B_BE_EF_ENT BIT(31) #define B_BE_EF_TCOLUMN_EN BIT(29) @@ -3852,6 +4033,18 @@ #define B_BE_SYSON_DIS_PMCR_BE_WRMSK BIT(2) #define B_BE_SYSON_R_BE_ARB_MASK GENMASK(1, 0) +#define R_BE_PCIE_MIO_INTF 0x00E4 +#define B_BE_AON_MIO_EPHY_1K_SEL_MASK GENMASK(29, 24) +#define B_BE_PCIE_MIO_ADDR_PAGE_V1_MASK GENMASK(20, 16) +#define B_BE_PCIE_MIO_ASIF BIT(15) +#define B_BE_PCIE_MIO_BYIOREG BIT(13) +#define B_BE_PCIE_MIO_RE BIT(12) +#define B_BE_PCIE_MIO_WE_MASK GENMASK(11, 8) +#define B_BE_PCIE_MIO_ADDR_MASK GENMASK(7, 0) + +#define R_BE_PCIE_MIO_INTD 0x00E8 +#define B_BE_PCIE_MIO_DATA_MASK GENMASK(31, 0) + #define R_BE_HALT_H2C_CTRL 0x0160 #define B_BE_HALT_H2C_TRIGGER BIT(0) @@ -3997,6 +4190,67 @@ #define R_BE_UDM2 0x01F8 #define B_BE_UDM2_EPC_RA_MASK GENMASK(31, 0) +#define R_BE_AFE_ON_CTRL0 0x0240 +#define B_BE_REG_LPF_R3_3_0_MASK GENMASK(31, 29) +#define B_BE_REG_LPF_R2_MASK GENMASK(28, 24) +#define B_BE_REG_LPF_C3_MASK GENMASK(23, 21) +#define B_BE_REG_LPF_C2_MASK GENMASK(20, 18) +#define B_BE_REG_LPF_C1_MASK GENMASK(17, 15) +#define B_BE_REG_CP_ICPX2 BIT(14) +#define B_BE_REG_CP_ICP_SEL_FAST_MASK GENMASK(13, 10) +#define B_BE_REG_CP_ICP_SEL_MASK GENMASK(9, 6) +#define B_BE_REG_IB_PI_MASK GENMASK(5, 4) +#define B_BE_REG_CK_DEBUG_BT BIT(3) +#define B_BE_EN_PC_LDO BIT(2) +#define B_BE_LDO_VSEL_MASK GENMASK(1, 0) + +#define R_BE_AFE_ON_CTRL1 0x0244 +#define B_BE_REG_CK_MON_SEL_MASK GENMASK(31, 29) +#define B_BE_REG_CK_MON_CK960M_EN BIT(28) +#define B_BE_REG_XTAL_FREQ_SEL BIT(27) +#define B_BE_REG_XTAL_EDGE_SEL BIT(26) +#define B_BE_REG_VCO_KVCO BIT(25) +#define B_BE_REG_SDM_EDGE_SEL BIT(24) +#define B_BE_REG_SDM_CK_SEL BIT(23) +#define B_BE_REG_SDM_CK_GATED BIT(22) +#define B_BE_REG_PFD_RESET_GATED BIT(21) +#define B_BE_REG_LPF_R3_FAST_MASK GENMASK(20, 16) +#define B_BE_REG_LPF_R2_FAST_MASK GENMASK(15, 11) +#define B_BE_REG_LPF_C3_FAST_MASK GENMASK(10, 8) +#define B_BE_REG_LPF_C2_FAST_MASK GENMASK(7, 5) +#define B_BE_REG_LPF_C1_FAST_MASK GENMASK(4, 2) +#define B_BE_REG_LPF_R3_4_MASK GENMASK(1, 0) + +#define R_BE_AFE_ON_CTRL3 0x024C +#define B_BE_LDO_VSEL_DA_1_MASK GENMASK(31, 30) +#define B_BE_LDO_VSEL_DA_0_MASK GENMASK(29, 28) +#define B_BE_LDO_VSEL_D2S_1_MASK GENMASK(27, 26) +#define B_BE_LDO_VSEL_D2S_0_MASK GENMASK(25, 24) +#define B_BE_LDO_VSEL_BUF_MASK GENMASK(23, 22) +#define B_BE_REG_R2_L_MASK GENMASK(21, 19) +#define B_BE_REG_R1_L_MASK GENMASK(18, 16) +#define B_BE_REG_CK_DEBUG_BT_MON BIT(15) +#define B_BE_REG_BT_CLK_BUF_POWER BIT(14) +#define B_BE_REG_BG_OUT_BTADC_V1 BIT(13) +#define B_BE_REG_SEL_V18 BIT(11) +#define B_BE_REG_FRAC_EN BIT(10) +#define B_BE_REG_CK1920M_EN BIT(9) +#define B_BE_REG_CK1280M_EN BIT(8) +#define B_BE_REG_12LDO_SEL_MASK GENMASK(7, 6) +#define B_BE_REG_09LDO_SEL_MASK GENMASK(5, 4) +#define B_BE_REG_VC_TH BIT(3) +#define B_BE_REG_VC_TL BIT(2) +#define B_BE_REG_CK40M_EN BIT(1) +#define B_BE_REG_CK640M_EN BIT(0) + +#define R_BE_WLAN_XTAL_SI_CTRL 0x0270 +#define B_BE_WL_XTAL_SI_CMD_POLL BIT(31) +#define B_BE_WL_XTAL_SI_CHIPID_MASK GENMASK(30, 28) +#define B_BE_WL_XTAL_SI_MODE_MASK GENMASK(25, 24) +#define B_BE_WL_XTAL_SI_BITMASK_MASK GENMASK(23, 16) +#define B_BE_WL_XTAL_SI_DATA_MASK GENMASK(15, 8) +#define B_BE_WL_XTAL_SI_ADDR_MASK GENMASK(7, 0) + #define R_BE_IC_PWR_STATE 0x03F0 #define B_BE_WHOLE_SYS_PWR_STE_MASK GENMASK(25, 16) #define MAC_AX_SYS_ACT 0x220 @@ -4006,6 +4260,8 @@ #define B_BE_USB_HCISYS_PWR_STE_MASK GENMASK(3, 2) #define B_BE_PCIE_HCISYS_PWR_STE_MASK GENMASK(1, 0) +#define R_BE_WLCPU_PORT_PC 0x03FC + #define R_BE_DCPU_PLATFORM_ENABLE 0x0888 #define B_BE_DCPU_SYM_DPLT_MEM_MUX_EN BIT(10) #define B_BE_DCPU_WARM_EN BIT(9) @@ -4015,6 +4271,34 @@ #define B_BE_DCPU_EN BIT(1) #define B_BE_DCPU_PLATFORM_EN BIT(0) +#define R_BE_PL_AXIDMA_IDCT_MSK 0x0910 +#define B_BE_PL_AXIDMA_RRESP_ERR_MASK BIT(6) +#define B_BE_PL_AXIDMA_BRESP_ERR_MASK BIT(5) +#define B_BE_PL_AXIDMA_FC_ERR_MASK BIT(4) +#define B_BE_PL_AXIDMA_TXBD_LEN0_MASK BIT(3) +#define B_BE_PL_AXIDMA_TXBD_4KBOUD_LENERR_MASK BIT(2) +#define B_BE_PL_AXIDMA_TXBD_RX_STUCK_MASK BIT(1) +#define B_BE_PL_AXIDMA_TXBD_TX_STUCK_MASK BIT(0) +#define B_BE_PL_AXIDMA_IDCT_MSK_CLR (B_BE_PL_AXIDMA_TXBD_TX_STUCK_MASK | \ + B_BE_PL_AXIDMA_TXBD_RX_STUCK_MASK | \ + B_BE_PL_AXIDMA_TXBD_LEN0_MASK | \ + B_BE_PL_AXIDMA_FC_ERR_MASK | \ + B_BE_PL_AXIDMA_BRESP_ERR_MASK | \ + B_BE_PL_AXIDMA_RRESP_ERR_MASK) +#define B_BE_PL_AXIDMA_IDCT_MSK_SET (B_BE_PL_AXIDMA_TXBD_TX_STUCK_MASK | \ + B_BE_PL_AXIDMA_TXBD_RX_STUCK_MASK | \ + B_BE_PL_AXIDMA_TXBD_LEN0_MASK | \ + B_BE_PL_AXIDMA_FC_ERR_MASK) + +#define R_BE_PL_AXIDMA_IDCT 0x0914 +#define B_BE_PL_AXIDMA_RRESP_ERR BIT(6) +#define B_BE_PL_AXIDMA_BRESP_ERR BIT(5) +#define B_BE_PL_AXIDMA_FC_ERR BIT(4) +#define B_BE_PL_AXIDMA_TXBD_LEN0 BIT(3) +#define B_BE_PL_AXIDMA_TXBD_4KBOUD_LENERR BIT(2) +#define B_BE_PL_AXIDMA_TXBD_RX_STUCK BIT(1) +#define B_BE_PL_AXIDMA_TXBD_TX_STUCK BIT(0) + #define R_BE_FILTER_MODEL_ADDR 0x0C04 #define R_BE_WLAN_WDT 0x3050 @@ -4123,6 +4407,53 @@ #define B_BE_HCI_RXDMA_EN BIT(1) #define B_BE_HCI_TXDMA_EN BIT(0) +#define R_BE_DMAC_FUNC_EN 0x8400 +#define B_BE_DMAC_CRPRT BIT(31) +#define B_BE_MAC_FUNC_EN BIT(30) +#define B_BE_DMAC_FUNC_EN BIT(29) +#define B_BE_MPDU_PROC_EN BIT(28) +#define B_BE_WD_RLS_EN BIT(27) +#define B_BE_DLE_WDE_EN BIT(26) +#define B_BE_TXPKT_CTRL_EN BIT(25) +#define B_BE_STA_SCH_EN BIT(24) +#define B_BE_DLE_PLE_EN BIT(23) +#define B_BE_PKT_BUF_EN BIT(22) +#define B_BE_DMAC_TBL_EN BIT(21) +#define B_BE_PKT_IN_EN BIT(20) +#define B_BE_DLE_CPUIO_EN BIT(19) +#define B_BE_DISPATCHER_EN BIT(18) +#define B_BE_BBRPT_EN BIT(17) +#define B_BE_MAC_SEC_EN BIT(16) +#define B_BE_DMACREG_GCKEN BIT(15) +#define B_BE_H_AXIDMA_EN BIT(14) +#define B_BE_DMAC_MLO_EN BIT(11) +#define B_BE_PLRLS_EN BIT(10) +#define B_BE_P_AXIDMA_EN BIT(9) +#define B_BE_DLE_DATACPUIO_EN BIT(8) +#define B_BE_LTR_CTL_EN BIT(7) + +#define R_BE_DMAC_CLK_EN 0x8404 +#define B_BE_MAC_CKEN BIT(30) +#define B_BE_DMAC_CKEN BIT(29) +#define B_BE_MPDU_CKEN BIT(28) +#define B_BE_WD_RLS_CLK_EN BIT(27) +#define B_BE_DLE_WDE_CLK_EN BIT(26) +#define B_BE_TXPKT_CTRL_CLK_EN BIT(25) +#define B_BE_STA_SCH_CLK_EN BIT(24) +#define B_BE_DLE_PLE_CLK_EN BIT(23) +#define B_BE_PKTBUF_CKEN BIT(22) +#define B_BE_DMAC_TABLE_CLK_EN BIT(21) +#define B_BE_PKT_IN_CLK_EN BIT(20) +#define B_BE_DLE_CPUIO_CLK_EN BIT(19) +#define B_BE_DISPATCHER_CLK_EN BIT(18) +#define B_BE_BBRPT_CLK_EN BIT(17) +#define B_BE_MAC_SEC_CLK_EN BIT(16) +#define B_BE_H_AXIDMA_CKEN BIT(14) +#define B_BE_DMAC_MLO_CKEN BIT(11) +#define B_BE_PLRLS_CKEN BIT(10) +#define B_BE_P_AXIDMA_CKEN BIT(9) +#define B_BE_DLE_DATACPUIO_CKEN BIT(8) + #define R_BE_LTR_CTRL_0 0x8410 #define B_BE_LTR_REQ_FW BIT(18) #define B_BE_LTR_IDX_FW_MASK GENMASK(17, 16) @@ -4144,6 +4475,725 @@ #define B_BE_LTR_CMAC1_RX_USE_PG_TH_MASK GENMASK(27, 16) #define B_BE_LTR_CMAC0_RX_USE_PG_TH_MASK GENMASK(11, 0) +#define R_BE_DMAC_TABLE_CTRL 0x8420 +#define B_BE_HWAMSDU_PADDING_MODE BIT(31) +#define B_BE_MACID_MPDU_PROCESSOR_OFFSET_MASK GENMASK(26, 16) +#define B_BE_DMAC_ADDR_MODE BIT(12) +#define B_BE_DMAC_CTRL_INFO_SER_IO BIT(11) +#define B_BE_DMAC_CTRL_INFO_OFFSET_MASK GENMASK(10, 0) + +#define R_BE_SER_DBG_INFO 0x8424 +#define B_BE_SER_L0_PROMOTE_L1_EVENT_MASK GENMASK(31, 28) +#define B_BE_SER_L1_COUNTER_MASK GENMASK(27, 24) +#define B_BE_RMAC_PPDU_HANG_CNT_MASK GENMASK(23, 16) +#define B_BE_SER_L0_COUNTER_MASK GENMASK(8, 0) + +#define R_BE_DLE_EMPTY0 0x8430 +#define B_BE_PLE_EMPTY_QTA_DMAC_H2D BIT(27) +#define B_BE_PLE_EMPTY_QTA_DMAC_CPUIO BIT(26) +#define B_BE_PLE_EMPTY_QTA_DMAC_MPDU_TX BIT(25) +#define B_BE_PLE_EMPTY_QTA_DMAC_WLAN_CPU BIT(24) +#define B_BE_PLE_EMPTY_QTA_DMAC_H2C BIT(23) +#define B_BE_PLE_EMPTY_QTA_DMAC_B1_TXPL BIT(22) +#define B_BE_PLE_EMPTY_QTA_DMAC_B0_TXPL BIT(21) +#define B_BE_WDE_EMPTY_QTA_DMAC_CPUIO BIT(20) +#define B_BE_WDE_EMPTY_QTA_DMAC_PKTIN BIT(19) +#define B_BE_WDE_EMPTY_QTA_DMAC_DATA_CPU BIT(18) +#define B_BE_WDE_EMPTY_QTA_DMAC_WLAN_CPU BIT(17) +#define B_BE_WDE_EMPTY_QTA_DMAC_HIF BIT(16) +#define B_BE_WDE_EMPTY_QUE_CMAC_B1_HIQ BIT(15) +#define B_BE_WDE_EMPTY_QUE_CMAC_B1_MBH BIT(14) +#define B_BE_WDE_EMPTY_QUE_CMAC_B0_OTHERS BIT(13) +#define B_BE_WDE_EMPTY_QUE_DMAC_MLO_ACQ BIT(12) +#define B_BE_WDE_EMPTY_QUE_DMAC_MLO_MISC BIT(11) +#define B_BE_WDE_EMPTY_QUE_DMAC_PKTIN BIT(10) +#define B_BE_PLE_EMPTY_QUE_DMAC_SEC_TX BIT(9) +#define B_BE_PLE_EMPTY_QUE_DMAC_MPDU_TX BIT(8) +#define B_BE_WDE_EMPTY_QUE_OTHERS BIT(7) +#define B_BE_WDE_EMPTY_QUE_CMAC_WMM3 BIT(6) +#define B_BE_WDE_EMPTY_QUE_CMAC_WMM2 BIT(5) +#define B_BE_WDE_EMPTY_QUE_CMAC0_WMM1 BIT(4) +#define B_BE_WDE_EMPTY_QUE_CMAC0_WMM0 BIT(3) +#define B_BE_WDE_EMPTY_QUE_CMAC1_MBH BIT(2) +#define B_BE_WDE_EMPTY_QUE_CMAC0_MBH BIT(1) +#define B_BE_WDE_EMPTY_QUE_CMAC0_ALL_AC BIT(0) + +#define R_BE_DLE_EMPTY1 0x8434 +#define B_BE_PLE_EMPTY_QTA_CMAC_DMA_TXRPT BIT(21) +#define B_BE_PLE_EMPTY_QTA_DMAC_WDRLS BIT(20) +#define B_BE_PLE_EMPTY_QTA_CMAC1_DMA_BBRPT BIT(19) +#define B_BE_PLE_EMPTY_QTA_CMAC1_DMA_RX BIT(18) +#define B_BE_PLE_EMPTY_QTA_CMAC0_DMA_RX BIT(17) +#define B_BE_PLE_EMPTY_QTA_DMAC_C2H BIT(16) +#define B_BE_PLE_EMPTY_QUE_DMAC_PLRLS BIT(5) +#define B_BE_PLE_EMPTY_QUE_DMAC_CPUIO BIT(4) +#define B_BE_PLE_EMPTY_QUE_DMAC_SEC_RX BIT(3) +#define B_BE_PLE_EMPTY_QUE_DMAC_MPDU_RX BIT(2) +#define B_BE_PLE_EMPTY_QUE_DMAC_HDP BIT(1) +#define B_BE_WDE_EMPTY_QUE_DMAC_WDRLS BIT(0) + +#define R_BE_SER_L1_DBG_CNT_0 0x8440 +#define B_BE_SER_L1_WDRLS_CNT_MASK GENMASK(31, 24) +#define B_BE_SER_L1_SEC_CNT_MASK GENMASK(23, 16) +#define B_BE_SER_L1_MPDU_CNT_MASK GENMASK(15, 8) +#define B_BE_SER_L1_STA_SCH_CNT_MASK GENMASK(7, 0) + +#define R_BE_SER_L1_DBG_CNT_1 0x8444 +#define B_BE_SER_L1_WDE_CNT_MASK GENMASK(31, 24) +#define B_BE_SER_L1_TXPKTCTRL_CNT_MASK GENMASK(23, 16) +#define B_BE_SER_L1_PLE_CNT_MASK GENMASK(15, 8) +#define B_BE_SER_L1_PKTIN_CNT_MASK GENMASK(7, 0) + +#define R_BE_SER_L1_DBG_CNT_2 0x8448 +#define B_BE_SER_L1_DISP_CNT_MASK GENMASK(31, 24) +#define B_BE_SER_L1_APB_BRIDGE_CNT_MASK GENMASK(23, 16) +#define B_BE_SER_L1_DLE_W_CPUIO_CNT_MASK GENMASK(15, 8) +#define B_BE_SER_L1_BBRPT_CNT_MASK GENMASK(7, 0) + +#define R_BE_SER_L1_DBG_CNT_3 0x844C +#define B_BE_SER_L1_HCI_BUF_CNT_MASK GENMASK(31, 24) +#define B_BE_SER_L1_P_AXIDMA_CNT_MASK GENMASK(23, 16) +#define B_BE_SER_L1_H_AXIDMA_CNT_MASK GENMASK(15, 8) +#define B_BE_SER_L1_MLO_ERR_CNT_MASK GENMASK(7, 0) + +#define R_BE_SER_L1_DBG_CNT_4 0x8450 +#define B_BE_SER_L1_PLDRLS_ERR_CNT_MASK GENMASK(31, 24) +#define B_BE_SER_L1_DLE_D_CPUIO_CNT_MASK GENMASK(23, 16) + +#define R_BE_SER_L1_DBG_CNT_5 0x8454 +#define B_BE_SER_L1_DBG_0_MASK GENMASK(31, 0) + +#define R_BE_SER_L1_DBG_CNT_6 0x8458 +#define B_BE_SER_L1_DBG_1_MASK GENMASK(31, 0) + +#define R_BE_SER_L1_DBG_CNT_7 0x845C +#define B_BE_SER_L1_DBG_2_MASK GENMASK(31, 0) + +#define R_BE_DMAC_ERR_IMR 0x8520 +#define B_BE_DMAC_NOTX_ERR_INT_EN BIT(21) +#define B_BE_DMAC_NORX_ERR_INT_EN BIT(20) +#define B_BE_DLE_DATACPUIO_ERR_INT_EN BIT(19) +#define B_BE_PLRSL_ERR_INT_EN BIT(18) +#define B_BE_MLO_ERR_INT_EN BIT(17) +#define B_BE_DMAC_FW_ERR_INT_EN BIT(16) +#define B_BE_H_AXIDMA_ERR_INT_EN BIT(14) +#define B_BE_P_AXIDMA_ERR_INT_EN BIT(13) +#define B_BE_HCI_BUF_ERR_INT_EN BIT(12) +#define B_BE_BBRPT_ERR_INT_EN BIT(11) +#define B_BE_DLE_CPUIO_ERR_INT_EN BIT(10) +#define B_BE_APB_BRIDGE_ERR_INT_EN BIT(9) +#define B_BE_DISPATCH_ERR_INT_EN BIT(8) +#define B_BE_PKTIN_ERR_INT_EN BIT(7) +#define B_BE_PLE_DLE_ERR_INT_EN BIT(6) +#define B_BE_TXPKTCTRL_ERR_INT_EN BIT(5) +#define B_BE_WDE_DLE_ERR_INT_EN BIT(4) +#define B_BE_STA_SCHEDULER_ERR_INT_EN BIT(3) +#define B_BE_MPDU_ERR_INT_EN BIT(2) +#define B_BE_WSEC_ERR_INT_EN BIT(1) +#define B_BE_WDRLS_ERR_INT_EN BIT(0) + +#define R_BE_DMAC_ERR_ISR 0x8524 +#define B_BE_DLE_DATACPUIO_ERR_INT BIT(19) +#define B_BE_PLRLS_ERR_INT BIT(18) +#define B_BE_MLO_ERR_INT BIT(17) +#define B_BE_DMAC_FW_ERR_IDCT BIT(16) +#define B_BE_H_AXIDMA_ERR_INT BIT(14) +#define B_BE_P_AXIDMA_ERR_INT BIT(13) +#define B_BE_HCI_BUF_ERR_FLAG BIT(12) +#define B_BE_BBRPT_ERR_FLAG BIT(11) +#define B_BE_DLE_CPUIO_ERR_FLAG BIT(10) +#define B_BE_APB_BRIDGE_ERR_FLAG BIT(9) +#define B_BE_DISPATCH_ERR_FLAG BIT(8) +#define B_BE_PKTIN_ERR_FLAG BIT(7) +#define B_BE_PLE_DLE_ERR_FLAG BIT(6) +#define B_BE_TXPKTCTRL_ERR_FLAG BIT(5) +#define B_BE_WDE_DLE_ERR_FLAG BIT(4) +#define B_BE_STA_SCHEDULER_ERR_FLAG BIT(3) +#define B_BE_MPDU_ERR_FLAG BIT(2) +#define B_BE_WSEC_ERR_FLAG BIT(1) +#define B_BE_WDRLS_ERR_FLAG BIT(0) + +#define R_BE_DISP_ERROR_ISR0 0x8804 +#define B_BE_REUSE_SIZE_ERR BIT(31) +#define B_BE_REUSE_EN_ERR BIT(30) +#define B_BE_STF_OQT_UNDERFLOW_ERR BIT(29) +#define B_BE_STF_OQT_OVERFLOW_ERR BIT(28) +#define B_BE_STF_WRFF_UNDERFLOW_ERR BIT(27) +#define B_BE_STF_WRFF_OVERFLOW_ERR BIT(26) +#define B_BE_STF_CMD_UNDERFLOW_ERR BIT(25) +#define B_BE_STF_CMD_OVERFLOW_ERR BIT(24) +#define B_BE_REUSE_SIZE_ZERO_ERR BIT(23) +#define B_BE_REUSE_PKT_CNT_ERR BIT(22) +#define B_BE_CDT_PTR_TIMEOUT_ERR BIT(21) +#define B_BE_CDT_HCI_TIMEOUT_ERR BIT(20) +#define B_BE_HDT_PTR_TIMEOUT_ERR BIT(19) +#define B_BE_HDT_HCI_TIMEOUT_ERR BIT(18) +#define B_BE_CDT_ADDR_INFO_LEN_ERR BIT(17) +#define B_BE_HDT_ADDR_INFO_LEN_ERR BIT(16) +#define B_BE_CDR_DMA_TIMEOUT_ERR BIT(15) +#define B_BE_CDR_RX_TIMEOUT_ERR BIT(14) +#define B_BE_PLE_OUTPUT_ERR BIT(12) +#define B_BE_PLE_RESPOSE_ERR BIT(11) +#define B_BE_PLE_BURST_NUM_ERR BIT(10) +#define B_BE_PLE_NULL_PKT_ERR BIT(9) +#define B_BE_PLE_FLOW_CTRL_ERR BIT(8) +#define B_BE_HDR_DMA_TIMEOUT_ERR BIT(7) +#define B_BE_HDR_RX_TIMEOUT_ERR BIT(6) +#define B_BE_WDE_OUTPUT_ERR BIT(4) +#define B_BE_WDE_RESPONSE_ERR BIT(3) +#define B_BE_WDE_BURST_NUM_ERR BIT(2) +#define B_BE_WDE_NULL_PKT_ERR BIT(1) +#define B_BE_WDE_FLOW_CTRL_ERR BIT(0) + +#define R_BE_DISP_ERROR_ISR1 0x8808 +#define B_BE_HR_WRFF_UNDERFLOW_ERR BIT(31) +#define B_BE_HR_WRFF_OVERFLOW_ERR BIT(30) +#define B_BE_HR_CHKSUM_FSM_ERR BIT(29) +#define B_BE_HR_SHIFT_DMA_CFG_ERR BIT(28) +#define B_BE_HR_DMA_PROCESS_ERR BIT(27) +#define B_BE_HR_TOTAL_LEN_UNDER_ERR BIT(26) +#define B_BE_HR_SHIFT_EN_ERR BIT(25) +#define B_BE_HR_AGG_CFG_ERR BIT(24) +#define B_BE_HR_PLD_LEN_ZERO_ERR BIT(22) +#define B_BE_HT_ILL_CH_ERR BIT(20) +#define B_BE_HT_ADDR_INFO_LEN_ERR BIT(18) +#define B_BE_HT_WD_LEN_OVER_ERR BIT(17) +#define B_BE_HT_PLD_CMD_UNDERFLOW_ERR BIT(16) +#define B_BE_HT_PLD_CMD_OVERFLOW_ERR BIT(15) +#define B_BE_HT_WRFF_UNDERFLOW_ERR BIT(14) +#define B_BE_HT_WRFF_OVERFLOW_ERR BIT(13) +#define B_BE_HT_CHKSUM_FSM_ERR BIT(12) +#define B_BE_HT_NON_IDLE_PKT_STR_ERR BIT(11) +#define B_BE_HT_PRE_SUB_BE_ERR BIT(10) +#define B_BE_HT_WD_CHKSUM_ERR BIT(9) +#define B_BE_HT_CHANNEL_DMA_ERR BIT(8) +#define B_BE_HT_OFFSET_UNMATCH_ERR BIT(7) +#define B_BE_HT_PAYLOAD_UNDER_ERR BIT(6) +#define B_BE_HT_PAYLOAD_OVER_ERR BIT(5) +#define B_BE_HT_PERMU_FF_UNDERFLOW_ERR BIT(4) +#define B_BE_HT_PERMU_FF_OVERFLOW_ERR BIT(3) +#define B_BE_HT_PKT_FAIL_ERR BIT(2) +#define B_BE_HT_CH_ID_ERR BIT(1) +#define B_BE_HT_EP_CH_DIFF_ERR BIT(0) + +#define R_BE_DISP_ERROR_ISR2 0x880C +#define B_BE_CR_PLD_LEN_ERR BIT(30) +#define B_BE_CR_WRFF_UNDERFLOW_ERR BIT(29) +#define B_BE_CR_WRFF_OVERFLOW_ERR BIT(28) +#define B_BE_CR_SHIFT_DMA_CFG_ERR BIT(27) +#define B_BE_CR_DMA_PROCESS_ERR BIT(26) +#define B_BE_CR_SHIFT_EN_ERR BIT(24) +#define B_BE_REUSE_FIFO_B_UNDER_ERR BIT(22) +#define B_BE_REUSE_FIFO_B_OVER_ERR BIT(21) +#define B_BE_REUSE_FIFO_A_UNDER_ERR BIT(20) +#define B_BE_REUSE_FIFO_A_OVER_ERR BIT(19) +#define B_BE_CT_ADDR_INFO_LEN_MISS_ERR BIT(17) +#define B_BE_CT_WD_LEN_OVER_ERR BIT(16) +#define B_BE_CT_F2P_SEQ_ERR BIT(15) +#define B_BE_CT_F2P_QSEL_ERR BIT(14) +#define B_BE_CT_PLD_CMD_UNDERFLOW_ERR BIT(13) +#define B_BE_CT_PLD_CMD_OVERFLOW_ERR BIT(12) +#define B_BE_CT_PRE_SUB_ERR BIT(11) +#define B_BE_CT_WD_CHKSUM_ERR BIT(10) +#define B_BE_CT_CHANNEL_DMA_ERR BIT(9) +#define B_BE_CT_OFFSET_UNMATCH_ERR BIT(8) +#define B_BE_F2P_TOTAL_NUM_ERR BIT(7) +#define B_BE_CT_PAYLOAD_UNDER_ERR BIT(6) +#define B_BE_CT_PAYLOAD_OVER_ERR BIT(5) +#define B_BE_CT_PERMU_FF_UNDERFLOW_ERR BIT(4) +#define B_BE_CT_PERMU_FF_OVERFLOW_ERR BIT(3) +#define B_BE_CT_CH_ID_ERR BIT(2) +#define B_BE_CT_EP_CH_DIFF_ERR BIT(0) + +#define R_BE_DISP_OTHER_IMR 0x8870 +#define B_BE_REUSE_SIZE_ERR_INT_EN BIT(31) +#define B_BE_REUSE_EN_ERR_INT_EN BIT(30) +#define B_BE_STF_OQT_UNDERFLOW_ERR_INT_EN BIT(29) +#define B_BE_STF_OQT_OVERFLOW_ERR_INT_EN BIT(28) +#define B_BE_STF_WRFF_UNDERFLOW_ERR_INT_EN BIT(27) +#define B_BE_STF_WRFF_OVERFLOW_ERR_INT_EN BIT(26) +#define B_BE_STF_CMD_UNDERFLOW_ERR_INT_EN BIT(25) +#define B_BE_STF_CMD_OVERFLOW_ERR_INT_EN BIT(24) +#define B_BE_REUSE_SIZE_ZERO_ERR_INT_EN BIT(23) +#define B_BE_REUSE_PKT_CNT_ERR_INT_EN BIT(22) +#define B_BE_CDT_PTR_TIMEOUT_ERR_INT_EN BIT(21) +#define B_BE_CDT_HCI_TIMEOUT_ERR_INT_EN BIT(20) +#define B_BE_HDT_PTR_TIMEOUT_ERR_INT_EN BIT(19) +#define B_BE_HDT_HCI_TIMEOUT_ERR_INT_EN BIT(18) +#define B_BE_CDT_ADDR_INFO_LEN_ERR_INT_EN BIT(17) +#define B_BE_HDT_ADDR_INFO_LEN_ERR_INT_EN BIT(16) +#define B_BE_CDR_DMA_TIMEOUT_ERR_INT_EN BIT(15) +#define B_BE_CDR_RX_TIMEOUT_ERR_INT_EN BIT(14) +#define B_BE_PLE_OUTPUT_ERR_INT_EN BIT(12) +#define B_BE_PLE_RESPOSE_ERR_INT_EN BIT(11) +#define B_BE_PLE_BURST_NUM_ERR_INT_EN BIT(10) +#define B_BE_PLE_NULL_PKT_ERR_INT_EN BIT(9) +#define B_BE_PLE_FLOW_CTRL_ERR_INT_EN BIT(8) +#define B_BE_HDR_DMA_TIMEOUT_ERR_INT_EN BIT(7) +#define B_BE_HDR_RX_TIMEOUT_ERR_INT_EN BIT(6) +#define B_BE_WDE_OUTPUT_ERR_INT_EN BIT(4) +#define B_BE_WDE_RESPONSE_ERR_INT_EN BIT(3) +#define B_BE_WDE_BURST_NUM_ERR_INT_EN BIT(2) +#define B_BE_WDE_NULL_PKT_ERR_INT_EN BIT(1) +#define B_BE_WDE_FLOW_CTRL_ERR_INT_EN BIT(0) +#define B_BE_DISP_OTHER_IMR_CLR (B_BE_WDE_FLOW_CTRL_ERR_INT_EN | \ + B_BE_WDE_NULL_PKT_ERR_INT_EN | \ + B_BE_WDE_BURST_NUM_ERR_INT_EN | \ + B_BE_WDE_RESPONSE_ERR_INT_EN | \ + B_BE_WDE_OUTPUT_ERR_INT_EN | \ + B_BE_HDR_RX_TIMEOUT_ERR_INT_EN | \ + B_BE_HDR_DMA_TIMEOUT_ERR_INT_EN | \ + B_BE_PLE_FLOW_CTRL_ERR_INT_EN | \ + B_BE_PLE_NULL_PKT_ERR_INT_EN | \ + B_BE_PLE_BURST_NUM_ERR_INT_EN | \ + B_BE_PLE_RESPOSE_ERR_INT_EN | \ + B_BE_PLE_OUTPUT_ERR_INT_EN | \ + B_BE_CDR_RX_TIMEOUT_ERR_INT_EN | \ + B_BE_CDR_DMA_TIMEOUT_ERR_INT_EN | \ + B_BE_HDT_ADDR_INFO_LEN_ERR_INT_EN | \ + B_BE_CDT_ADDR_INFO_LEN_ERR_INT_EN | \ + B_BE_HDT_HCI_TIMEOUT_ERR_INT_EN | \ + B_BE_HDT_PTR_TIMEOUT_ERR_INT_EN | \ + B_BE_CDT_HCI_TIMEOUT_ERR_INT_EN | \ + B_BE_CDT_PTR_TIMEOUT_ERR_INT_EN | \ + B_BE_REUSE_PKT_CNT_ERR_INT_EN | \ + B_BE_REUSE_SIZE_ZERO_ERR_INT_EN | \ + B_BE_STF_CMD_OVERFLOW_ERR_INT_EN | \ + B_BE_STF_CMD_UNDERFLOW_ERR_INT_EN | \ + B_BE_STF_WRFF_OVERFLOW_ERR_INT_EN | \ + B_BE_STF_WRFF_UNDERFLOW_ERR_INT_EN | \ + B_BE_STF_OQT_OVERFLOW_ERR_INT_EN | \ + B_BE_STF_OQT_UNDERFLOW_ERR_INT_EN | \ + B_BE_REUSE_EN_ERR_INT_EN | \ + B_BE_REUSE_SIZE_ERR_INT_EN) +#define B_BE_DISP_OTHER_IMR_SET (B_BE_STF_CMD_OVERFLOW_ERR_INT_EN | \ + B_BE_STF_CMD_UNDERFLOW_ERR_INT_EN | \ + B_BE_STF_WRFF_OVERFLOW_ERR_INT_EN | \ + B_BE_STF_WRFF_UNDERFLOW_ERR_INT_EN | \ + B_BE_STF_OQT_OVERFLOW_ERR_INT_EN | \ + B_BE_STF_OQT_UNDERFLOW_ERR_INT_EN) + +#define R_BE_DISP_HOST_IMR 0x8874 +#define B_BE_HR_WRFF_UNDERFLOW_ERR_INT_EN BIT(31) +#define B_BE_HR_WRFF_OVERFLOW_ERR_INT_EN BIT(30) +#define B_BE_HR_CHKSUM_FSM_ERR_INT_EN BIT(29) +#define B_BE_HR_SHIFT_DMA_CFG_ERR_INT_EN BIT(28) +#define B_BE_HR_DMA_PROCESS_ERR_INT_EN BIT(27) +#define B_BE_HR_TOTAL_LEN_UNDER_ERR_INT_EN BIT(26) +#define B_BE_HR_SHIFT_EN_ERR_INT_EN BIT(25) +#define B_BE_HR_AGG_CFG_ERR_INT_EN BIT(24) +#define B_BE_HR_PLD_LEN_ZERO_ERR_INT_EN BIT(22) +#define B_BE_HT_ILL_CH_ERR_INT_EN BIT(20) +#define B_BE_HT_ADDR_INFO_LEN_ERR_INT_EN BIT(18) +#define B_BE_HT_WD_LEN_OVER_ERR_INT_EN BIT(17) +#define B_BE_HT_PLD_CMD_UNDERFLOW_ERR_INT_EN BIT(16) +#define B_BE_HT_PLD_CMD_OVERFLOW_ERR_INT_EN BIT(15) +#define B_BE_HT_WRFF_UNDERFLOW_ERR_INT_EN BIT(14) +#define B_BE_HT_WRFF_OVERFLOW_ERR_INT_EN BIT(13) +#define B_BE_HT_CHKSUM_FSM_ERR_INT_EN BIT(12) +#define B_BE_HT_NON_IDLE_PKT_STR_ERR_EN BIT(11) +#define B_BE_HT_PRE_SUB_ERR_INT_EN BIT(10) +#define B_BE_HT_WD_CHKSUM_ERR_INT_EN BIT(9) +#define B_BE_HT_CHANNEL_DMA_ERR_INT_EN BIT(8) +#define B_BE_HT_OFFSET_UNMATCH_ERR_INT_EN BIT(7) +#define B_BE_HT_PAYLOAD_UNDER_ERR_INT_EN BIT(6) +#define B_BE_HT_PAYLOAD_OVER_ERR_INT_EN BIT(5) +#define B_BE_HT_PERMU_FF_UNDERFLOW_ERR_INT_EN BIT(4) +#define B_BE_HT_PERMU_FF_OVERFLOW_ERR_INT_EN BIT(3) +#define B_BE_HT_PKT_FAIL_ERR_INT_EN BIT(2) +#define B_BE_HT_CH_ID_ERR_INT_EN BIT(1) +#define B_BE_HT_EP_CH_DIFF_ERR_INT_EN BIT(0) +#define B_BE_DISP_HOST_IMR_CLR (B_BE_HT_EP_CH_DIFF_ERR_INT_EN | \ + B_BE_HT_CH_ID_ERR_INT_EN | \ + B_BE_HT_PKT_FAIL_ERR_INT_EN | \ + B_BE_HT_PERMU_FF_OVERFLOW_ERR_INT_EN | \ + B_BE_HT_PERMU_FF_UNDERFLOW_ERR_INT_EN | \ + B_BE_HT_PAYLOAD_OVER_ERR_INT_EN | \ + B_BE_HT_PAYLOAD_UNDER_ERR_INT_EN | \ + B_BE_HT_OFFSET_UNMATCH_ERR_INT_EN | \ + B_BE_HT_CHANNEL_DMA_ERR_INT_EN | \ + B_BE_HT_WD_CHKSUM_ERR_INT_EN | \ + B_BE_HT_PRE_SUB_ERR_INT_EN | \ + B_BE_HT_NON_IDLE_PKT_STR_ERR_EN | \ + B_BE_HT_CHKSUM_FSM_ERR_INT_EN | \ + B_BE_HT_WRFF_OVERFLOW_ERR_INT_EN | \ + B_BE_HT_WRFF_UNDERFLOW_ERR_INT_EN | \ + B_BE_HT_PLD_CMD_OVERFLOW_ERR_INT_EN | \ + B_BE_HT_PLD_CMD_UNDERFLOW_ERR_INT_EN | \ + B_BE_HT_WD_LEN_OVER_ERR_INT_EN | \ + B_BE_HT_ADDR_INFO_LEN_ERR_INT_EN | \ + B_BE_HT_ILL_CH_ERR_INT_EN | \ + B_BE_HR_PLD_LEN_ZERO_ERR_INT_EN | \ + B_BE_HR_AGG_CFG_ERR_INT_EN | \ + B_BE_HR_SHIFT_EN_ERR_INT_EN | \ + B_BE_HR_TOTAL_LEN_UNDER_ERR_INT_EN | \ + B_BE_HR_DMA_PROCESS_ERR_INT_EN | \ + B_BE_HR_SHIFT_DMA_CFG_ERR_INT_EN | \ + B_BE_HR_CHKSUM_FSM_ERR_INT_EN | \ + B_BE_HR_WRFF_OVERFLOW_ERR_INT_EN | \ + B_BE_HR_WRFF_UNDERFLOW_ERR_INT_EN) +#define B_BE_DISP_HOST_IMR_SET (B_BE_HT_EP_CH_DIFF_ERR_INT_EN | \ + B_BE_HT_PERMU_FF_OVERFLOW_ERR_INT_EN | \ + B_BE_HT_PERMU_FF_UNDERFLOW_ERR_INT_EN | \ + B_BE_HT_PAYLOAD_OVER_ERR_INT_EN | \ + B_BE_HT_PAYLOAD_UNDER_ERR_INT_EN | \ + B_BE_HT_CHANNEL_DMA_ERR_INT_EN | \ + B_BE_HT_PRE_SUB_ERR_INT_EN | \ + B_BE_HT_WRFF_OVERFLOW_ERR_INT_EN | \ + B_BE_HT_WRFF_UNDERFLOW_ERR_INT_EN | \ + B_BE_HT_PLD_CMD_OVERFLOW_ERR_INT_EN | \ + B_BE_HT_PLD_CMD_UNDERFLOW_ERR_INT_EN | \ + B_BE_HT_WD_LEN_OVER_ERR_INT_EN | \ + B_BE_HT_ILL_CH_ERR_INT_EN | \ + B_BE_HR_TOTAL_LEN_UNDER_ERR_INT_EN | \ + B_BE_HR_DMA_PROCESS_ERR_INT_EN | \ + B_BE_HR_WRFF_OVERFLOW_ERR_INT_EN | \ + B_BE_HR_WRFF_UNDERFLOW_ERR_INT_EN) + +#define R_BE_DISP_CPU_IMR 0x8878 +#define B_BE_CR_PLD_LEN_ERR_INT_EN BIT(30) +#define B_BE_CR_WRFF_UNDERFLOW_ERR_INT_EN BIT(29) +#define B_BE_CR_WRFF_OVERFLOW_ERR_INT_EN BIT(28) +#define B_BE_CR_SHIFT_DMA_CFG_ERR_INT_EN BIT(27) +#define B_BE_CR_DMA_PROCESS_ERR_INT_EN BIT(26) +#define B_BE_CR_TOTAL_LEN_UNDER_ERR_INT_EN BIT(25) +#define B_BE_CR_SHIFT_EN_ERR_INT_EN BIT(24) +#define B_BE_REUSE_FIFO_B_UNDER_ERR_INT_EN BIT(22) +#define B_BE_REUSE_FIFO_B_OVER_ERR_INT_EN BIT(21) +#define B_BE_REUSE_FIFO_A_UNDER_ERR_INT_EN BIT(20) +#define B_BE_REUSE_FIFO_A_OVER_ERR_INT_EN BIT(19) +#define B_BE_CT_ADDR_INFO_LEN_MISS_ERR_INT_EN BIT(17) +#define B_BE_CT_WD_LEN_OVER_ERR_INT_EN BIT(16) +#define B_BE_CT_F2P_SEQ_ERR_INT_EN BIT(15) +#define B_BE_CT_F2P_QSEL_ERR_INT_EN BIT(14) +#define B_BE_CT_PLD_CMD_UNDERFLOW_ERR_INT_EN BIT(13) +#define B_BE_CT_PLD_CMD_OVERFLOW_ERR_INT_EN BIT(12) +#define B_BE_CT_PRE_SUB_ERR_INT_EN BIT(11) +#define B_BE_CT_WD_CHKSUM_ERR_INT_EN BIT(10) +#define B_BE_CT_CHANNEL_DMA_ERR_INT_EN BIT(9) +#define B_BE_CT_OFFSET_UNMATCH_ERR_INT_EN BIT(8) +#define B_BE_CT_PAYLOAD_CHKSUM_ERR_INT_EN BIT(7) +#define B_BE_CT_PAYLOAD_UNDER_ERR_INT_EN BIT(6) +#define B_BE_CT_PAYLOAD_OVER_ERR_INT_EN BIT(5) +#define B_BE_CT_PERMU_FF_UNDERFLOW_ERR_INT_EN BIT(4) +#define B_BE_CT_PERMU_FF_OVERFLOW_ERR_INT_EN BIT(3) +#define B_BE_CT_CH_ID_ERR_INT_EN BIT(2) +#define B_BE_CT_PKT_FAIL_ERR_INT_EN BIT(1) +#define B_BE_CT_EP_CH_DIFF_ERR_INT_EN BIT(0) +#define B_BE_DISP_CPU_IMR_CLR (B_BE_CT_EP_CH_DIFF_ERR_INT_EN | \ + B_BE_CT_CH_ID_ERR_INT_EN | \ + B_BE_CT_PERMU_FF_OVERFLOW_ERR_INT_EN | \ + B_BE_CT_PERMU_FF_UNDERFLOW_ERR_INT_EN | \ + B_BE_CT_PAYLOAD_OVER_ERR_INT_EN | \ + B_BE_CT_PAYLOAD_UNDER_ERR_INT_EN | \ + B_BE_CT_OFFSET_UNMATCH_ERR_INT_EN | \ + B_BE_CT_CHANNEL_DMA_ERR_INT_EN | \ + B_BE_CT_WD_CHKSUM_ERR_INT_EN | \ + B_BE_CT_PRE_SUB_ERR_INT_EN | \ + B_BE_CT_PLD_CMD_OVERFLOW_ERR_INT_EN | \ + B_BE_CT_PLD_CMD_UNDERFLOW_ERR_INT_EN | \ + B_BE_CT_F2P_QSEL_ERR_INT_EN | \ + B_BE_CT_F2P_SEQ_ERR_INT_EN | \ + B_BE_CT_WD_LEN_OVER_ERR_INT_EN | \ + B_BE_CT_ADDR_INFO_LEN_MISS_ERR_INT_EN | \ + B_BE_REUSE_FIFO_A_OVER_ERR_INT_EN | \ + B_BE_REUSE_FIFO_A_UNDER_ERR_INT_EN | \ + B_BE_REUSE_FIFO_B_OVER_ERR_INT_EN | \ + B_BE_REUSE_FIFO_B_UNDER_ERR_INT_EN | \ + B_BE_CR_SHIFT_EN_ERR_INT_EN | \ + B_BE_CR_DMA_PROCESS_ERR_INT_EN | \ + B_BE_CR_SHIFT_DMA_CFG_ERR_INT_EN | \ + B_BE_CR_WRFF_OVERFLOW_ERR_INT_EN | \ + B_BE_CR_WRFF_UNDERFLOW_ERR_INT_EN | \ + B_BE_CR_PLD_LEN_ERR_INT_EN) +#define B_BE_DISP_CPU_IMR_SET (B_BE_CT_EP_CH_DIFF_ERR_INT_EN | \ + B_BE_CT_CH_ID_ERR_INT_EN | \ + B_BE_CT_PERMU_FF_OVERFLOW_ERR_INT_EN | \ + B_BE_CT_PERMU_FF_UNDERFLOW_ERR_INT_EN | \ + B_BE_CT_PAYLOAD_OVER_ERR_INT_EN | \ + B_BE_CT_PAYLOAD_UNDER_ERR_INT_EN | \ + B_BE_CT_PRE_SUB_ERR_INT_EN | \ + B_BE_CT_PLD_CMD_OVERFLOW_ERR_INT_EN | \ + B_BE_CT_PLD_CMD_UNDERFLOW_ERR_INT_EN | \ + B_BE_CT_WD_LEN_OVER_ERR_INT_EN | \ + B_BE_REUSE_FIFO_A_OVER_ERR_INT_EN | \ + B_BE_REUSE_FIFO_A_UNDER_ERR_INT_EN | \ + B_BE_REUSE_FIFO_B_OVER_ERR_INT_EN | \ + B_BE_REUSE_FIFO_B_UNDER_ERR_INT_EN | \ + B_BE_CR_DMA_PROCESS_ERR_INT_EN | \ + B_BE_CR_WRFF_OVERFLOW_ERR_INT_EN | \ + B_BE_CR_WRFF_UNDERFLOW_ERR_INT_EN) + +#define R_BE_WDE_PKTBUF_CFG 0x8C08 +#define B_BE_WDE_FREE_PAGE_NUM_MASK GENMASK(28, 16) +#define B_BE_WDE_START_BOUND_MASK GENMASK(14, 8) +#define B_BE_WDE_PAGE_SEL_MASK GENMASK(1, 0) + +#define R_BE_WDE_ERR_IMR 0x8C38 +#define B_BE_WDE_DATCHN_CAMREQ_ERR_INT_EN BIT(29) +#define B_BE_WDE_DATCHN_ADRERR_ERR_INT_EN BIT(28) +#define B_BE_WDE_DATCHN_RRDY_ERR_INT_EN BIT(27) +#define B_BE_WDE_DATCHN_FRZTO_ERR_INT_EN BIT(26) +#define B_BE_WDE_DATCHN_NULLPG_ERR_INT_EN BIT(25) +#define B_BE_WDE_DATCHN_ARBT_ERR_INT_EN BIT(24) +#define B_BE_WDE_QUEMGN_FRZTO_ERR_INT_EN BIT(23) +#define B_BE_WDE_NXTPKTLL_AD_ERR_INT_EN BIT(22) +#define B_BE_WDE_PREPKTLLT_AD_ERR_INT_EN BIT(21) +#define B_BE_WDE_ENQ_PKTCNT_NVAL_ERR_INT_EN BIT(20) +#define B_BE_WDE_ENQ_PKTCNT_OVRF_ERR_INT_EN BIT(19) +#define B_BE_WDE_QUE_SRCQUEID_ERR_INT_EN BIT(18) +#define B_BE_WDE_QUE_DSTQUEID_ERR_INT_EN BIT(17) +#define B_BE_WDE_QUE_CMDTYPE_ERR_INT_EN BIT(16) +#define B_BE_WDE_BUFMGN_MRG_SZLMT_ERR_INT_EN BIT(13) +#define B_BE_WDE_BUFMGN_MRG_QTAID_ERR_INT_EN BIT(12) +#define B_BE_WDE_BUFMGN_MRG_ENDPKTID_ERR_INT_EN BIT(11) +#define B_BE_WDE_ERR_BUFMGN_MRG_STRPKTID_ERR_INT_EN BIT(10) +#define B_BE_WDE_BUFMGN_FRZTO_ERR_INT_EN BIT(9) +#define B_BE_WDE_GETNPG_PGOFST_ERR_INT_EN BIT(8) +#define B_BE_WDE_GETNPG_STRPG_ERR_INT_EN BIT(7) +#define B_BE_WDE_BUFREQ_SRCHTAILPG_ERR_INT_EN BIT(6) +#define B_BE_WDE_BUFRTN_SIZE_ERR_INT_EN BIT(5) +#define B_BE_WDE_BUFRTN_INVLD_PKTID_ERR_INT_EN BIT(4) +#define B_BE_WDE_BUFREQ_UNAVAL_ERR_INT_EN BIT(3) +#define B_BE_WDE_BUFREQ_SIZELMT_INT_EN BIT(2) +#define B_BE_WDE_BUFREQ_SIZE0_INT_EN BIT(1) +#define B_BE_WDE_BUFREQ_QTAID_ERR_INT_EN BIT(0) +#define B_BE_WDE_ERR_IMR_CLR (B_BE_WDE_BUFREQ_QTAID_ERR_INT_EN | \ + B_BE_WDE_BUFREQ_SIZE0_INT_EN | \ + B_BE_WDE_BUFREQ_SIZELMT_INT_EN | \ + B_BE_WDE_BUFREQ_UNAVAL_ERR_INT_EN | \ + B_BE_WDE_BUFRTN_INVLD_PKTID_ERR_INT_EN | \ + B_BE_WDE_BUFRTN_SIZE_ERR_INT_EN | \ + B_BE_WDE_BUFREQ_SRCHTAILPG_ERR_INT_EN | \ + B_BE_WDE_GETNPG_STRPG_ERR_INT_EN | \ + B_BE_WDE_GETNPG_PGOFST_ERR_INT_EN | \ + B_BE_WDE_BUFMGN_FRZTO_ERR_INT_EN | \ + B_BE_WDE_ERR_BUFMGN_MRG_STRPKTID_ERR_INT_EN | \ + B_BE_WDE_BUFMGN_MRG_ENDPKTID_ERR_INT_EN | \ + B_BE_WDE_BUFMGN_MRG_QTAID_ERR_INT_EN | \ + B_BE_WDE_BUFMGN_MRG_SZLMT_ERR_INT_EN | \ + B_BE_WDE_QUE_CMDTYPE_ERR_INT_EN | \ + B_BE_WDE_QUE_DSTQUEID_ERR_INT_EN | \ + B_BE_WDE_QUE_SRCQUEID_ERR_INT_EN | \ + B_BE_WDE_ENQ_PKTCNT_OVRF_ERR_INT_EN | \ + B_BE_WDE_ENQ_PKTCNT_NVAL_ERR_INT_EN | \ + B_BE_WDE_PREPKTLLT_AD_ERR_INT_EN | \ + B_BE_WDE_NXTPKTLL_AD_ERR_INT_EN | \ + B_BE_WDE_QUEMGN_FRZTO_ERR_INT_EN | \ + B_BE_WDE_DATCHN_ARBT_ERR_INT_EN | \ + B_BE_WDE_DATCHN_NULLPG_ERR_INT_EN | \ + B_BE_WDE_DATCHN_FRZTO_ERR_INT_EN | \ + B_BE_WDE_DATCHN_RRDY_ERR_INT_EN | \ + B_BE_WDE_DATCHN_ADRERR_ERR_INT_EN | \ + B_BE_WDE_DATCHN_CAMREQ_ERR_INT_EN) +#define B_BE_WDE_ERR_IMR_SET (B_BE_WDE_BUFREQ_QTAID_ERR_INT_EN | \ + B_BE_WDE_BUFREQ_SIZE0_INT_EN | \ + B_BE_WDE_BUFREQ_SIZELMT_INT_EN | \ + B_BE_WDE_BUFREQ_UNAVAL_ERR_INT_EN | \ + B_BE_WDE_BUFRTN_INVLD_PKTID_ERR_INT_EN | \ + B_BE_WDE_BUFRTN_SIZE_ERR_INT_EN | \ + B_BE_WDE_BUFREQ_SRCHTAILPG_ERR_INT_EN | \ + B_BE_WDE_GETNPG_STRPG_ERR_INT_EN | \ + B_BE_WDE_GETNPG_PGOFST_ERR_INT_EN | \ + B_BE_WDE_BUFMGN_FRZTO_ERR_INT_EN | \ + B_BE_WDE_ERR_BUFMGN_MRG_STRPKTID_ERR_INT_EN | \ + B_BE_WDE_BUFMGN_MRG_ENDPKTID_ERR_INT_EN | \ + B_BE_WDE_BUFMGN_MRG_QTAID_ERR_INT_EN | \ + B_BE_WDE_BUFMGN_MRG_SZLMT_ERR_INT_EN | \ + B_BE_WDE_QUE_CMDTYPE_ERR_INT_EN | \ + B_BE_WDE_QUE_DSTQUEID_ERR_INT_EN | \ + B_BE_WDE_QUE_SRCQUEID_ERR_INT_EN | \ + B_BE_WDE_ENQ_PKTCNT_OVRF_ERR_INT_EN | \ + B_BE_WDE_ENQ_PKTCNT_NVAL_ERR_INT_EN | \ + B_BE_WDE_PREPKTLLT_AD_ERR_INT_EN | \ + B_BE_WDE_NXTPKTLL_AD_ERR_INT_EN | \ + B_BE_WDE_QUEMGN_FRZTO_ERR_INT_EN | \ + B_BE_WDE_DATCHN_ARBT_ERR_INT_EN | \ + B_BE_WDE_DATCHN_NULLPG_ERR_INT_EN | \ + B_BE_WDE_DATCHN_FRZTO_ERR_INT_EN | \ + B_BE_WDE_DATCHN_RRDY_ERR_INT_EN | \ + B_BE_WDE_DATCHN_ADRERR_ERR_INT_EN | \ + B_BE_WDE_DATCHN_CAMREQ_ERR_INT_EN) + +#define R_BE_WDE_QTA0_CFG 0x8C40 +#define B_BE_WDE_Q0_MAX_SIZE_MASK GENMASK(27, 16) +#define B_BE_WDE_Q0_MIN_SIZE_MASK GENMASK(11, 0) + +#define R_BE_WDE_QTA1_CFG 0x8C44 +#define B_BE_WDE_Q1_MAX_SIZE_MASK GENMASK(27, 16) +#define B_BE_WDE_Q1_MIN_SIZE_MASK GENMASK(11, 0) + +#define R_BE_WDE_QTA2_CFG 0x8C48 +#define B_BE_WDE_Q2_MAX_SIZE_MASK GENMASK(27, 16) +#define B_BE_WDE_Q2_MIN_SIZE_MASK GENMASK(11, 0) + +#define R_BE_WDE_QTA3_CFG 0x8C4C +#define B_BE_WDE_Q3_MAX_SIZE_MASK GENMASK(27, 16) +#define B_BE_WDE_Q3_MIN_SIZE_MASK GENMASK(11, 0) + +#define R_BE_WDE_QTA4_CFG 0x8C50 +#define B_BE_WDE_Q4_MAX_SIZE_MASK GENMASK(27, 16) +#define B_BE_WDE_Q4_MIN_SIZE_MASK GENMASK(11, 0) + +#define R_BE_WDE_ERR1_IMR 0x8CC0 +#define B_BE_WDE_QUEMGN_CMACACQ_DEQNTFY_INT_EN BIT(8) +#define B_BE_WDE_ERR1_IMR_CLR B_BE_WDE_QUEMGN_CMACACQ_DEQNTFY_INT_EN +#define B_BE_WDE_ERR1_IMR_SET B_BE_WDE_QUEMGN_CMACACQ_DEQNTFY_INT_EN + +#define R_BE_PLE_PKTBUF_CFG 0x9008 +#define B_BE_PLE_FREE_PAGE_NUM_MASK GENMASK(28, 16) +#define B_BE_PLE_START_BOUND_MASK GENMASK(14, 8) +#define B_BE_PLE_PAGE_SEL_MASK GENMASK(1, 0) + +#define R_BE_PLE_ERR_IMR 0x9038 +#define B_BE_PLE_DATCHN_CAMREQ_ERR_INT_EN BIT(29) +#define B_BE_PLE_DATCHN_ADRERR_ERR_INT_EN BIT(28) +#define B_BE_PLE_DATCHN_RRDY_ERR_INT_EN BIT(27) +#define B_BE_PLE_DATCHN_FRZTO_ERR_INT_EN BIT(26) +#define B_BE_PLE_DATCHN_NULLPG_ERR_INT_EN BIT(25) +#define B_BE_PLE_DATCHN_ARBT_ERR_INT_EN BIT(24) +#define B_BE_PLE_QUEMGN_FRZTO_ERR_INT_EN BIT(23) +#define B_BE_PLE_NXTPKTLL_AD_ERR_INT_EN BIT(22) +#define B_BE_PLE_PREPKTLLT_AD_ERR_INT_EN BIT(21) +#define B_BE_PLE_ENQ_PKTCNT_NVAL_ERR_INT_EN BIT(20) +#define B_BE_PLE_ENQ_PKTCNT_OVRF_ERR_INT_EN BIT(19) +#define B_BE_PLE_QUE_SRCQUEID_ERR_INT_EN BIT(18) +#define B_BE_PLE_QUE_DSTQUEID_ERR_INT_EN BIT(17) +#define B_BE_PLE_QUE_CMDTYPE_ERR_INT_EN BIT(16) +#define B_BE_PLE_BUFMGN_MRG_SZLMT_ERR_INT_EN BIT(13) +#define B_BE_PLE_BUFMGN_MRG_QTAID_ERR_INT_EN BIT(12) +#define B_BE_PLE_BUFMGN_MRG_ENDPKTID_ERR_INT_EN BIT(11) +#define B_BE_PLE_BUFMGN_MRG_STRPKTID_ERR_INT_EN BIT(10) +#define B_BE_PLE_BUFMGN_FRZTO_ERR_INT_EN BIT(9) +#define B_BE_PLE_GETNPG_PGOFST_ERR_INT_EN BIT(8) +#define B_BE_PLE_GETNPG_STRPG_ERR_INT_EN BIT(7) +#define B_BE_PLE_BUFREQ_SRCHTAILPG_ERR_INT_EN BIT(6) +#define B_BE_PLE_BUFRTN_SIZE_ERR_INT_EN BIT(5) +#define B_BE_PLE_BUFRTN_INVLD_PKTID_ERR_INT_EN BIT(4) +#define B_BE_PLE_BUFREQ_UNAVAL_ERR_INT_EN BIT(3) +#define B_BE_PLE_BUFREQ_SIZELMT_INT_EN BIT(2) +#define B_BE_PLE_BUFREQ_SIZE0_INT_EN BIT(1) +#define B_BE_PLE_BUFREQ_QTAID_ERR_INT_EN BIT(0) +#define B_BE_PLE_ERR_IMR_CLR (B_BE_PLE_BUFREQ_QTAID_ERR_INT_EN | \ + B_BE_PLE_BUFREQ_SIZE0_INT_EN | \ + B_BE_PLE_BUFREQ_SIZELMT_INT_EN | \ + B_BE_PLE_BUFREQ_UNAVAL_ERR_INT_EN | \ + B_BE_PLE_BUFRTN_INVLD_PKTID_ERR_INT_EN | \ + B_BE_PLE_BUFRTN_SIZE_ERR_INT_EN | \ + B_BE_PLE_BUFREQ_SRCHTAILPG_ERR_INT_EN | \ + B_BE_PLE_GETNPG_STRPG_ERR_INT_EN | \ + B_BE_PLE_GETNPG_PGOFST_ERR_INT_EN | \ + B_BE_PLE_BUFMGN_FRZTO_ERR_INT_EN | \ + B_BE_PLE_BUFMGN_MRG_STRPKTID_ERR_INT_EN | \ + B_BE_PLE_BUFMGN_MRG_ENDPKTID_ERR_INT_EN | \ + B_BE_PLE_BUFMGN_MRG_QTAID_ERR_INT_EN | \ + B_BE_PLE_BUFMGN_MRG_SZLMT_ERR_INT_EN | \ + B_BE_PLE_QUE_CMDTYPE_ERR_INT_EN | \ + B_BE_PLE_QUE_DSTQUEID_ERR_INT_EN | \ + B_BE_PLE_QUE_SRCQUEID_ERR_INT_EN | \ + B_BE_PLE_ENQ_PKTCNT_OVRF_ERR_INT_EN | \ + B_BE_PLE_ENQ_PKTCNT_NVAL_ERR_INT_EN | \ + B_BE_PLE_PREPKTLLT_AD_ERR_INT_EN | \ + B_BE_PLE_NXTPKTLL_AD_ERR_INT_EN | \ + B_BE_PLE_QUEMGN_FRZTO_ERR_INT_EN | \ + B_BE_PLE_DATCHN_ARBT_ERR_INT_EN | \ + B_BE_PLE_DATCHN_NULLPG_ERR_INT_EN | \ + B_BE_PLE_DATCHN_FRZTO_ERR_INT_EN | \ + B_BE_PLE_DATCHN_RRDY_ERR_INT_EN | \ + B_BE_PLE_DATCHN_ADRERR_ERR_INT_EN | \ + B_BE_PLE_DATCHN_CAMREQ_ERR_INT_EN) +#define B_BE_PLE_ERR_IMR_SET (B_BE_PLE_BUFREQ_QTAID_ERR_INT_EN | \ + B_BE_PLE_BUFREQ_SIZE0_INT_EN | \ + B_BE_PLE_BUFREQ_SIZELMT_INT_EN | \ + B_BE_PLE_BUFREQ_UNAVAL_ERR_INT_EN | \ + B_BE_PLE_BUFRTN_INVLD_PKTID_ERR_INT_EN | \ + B_BE_PLE_BUFRTN_SIZE_ERR_INT_EN | \ + B_BE_PLE_BUFREQ_SRCHTAILPG_ERR_INT_EN | \ + B_BE_PLE_GETNPG_STRPG_ERR_INT_EN | \ + B_BE_PLE_GETNPG_PGOFST_ERR_INT_EN | \ + B_BE_PLE_BUFMGN_FRZTO_ERR_INT_EN | \ + B_BE_PLE_BUFMGN_MRG_STRPKTID_ERR_INT_EN | \ + B_BE_PLE_BUFMGN_MRG_ENDPKTID_ERR_INT_EN | \ + B_BE_PLE_BUFMGN_MRG_QTAID_ERR_INT_EN | \ + B_BE_PLE_BUFMGN_MRG_SZLMT_ERR_INT_EN | \ + B_BE_PLE_QUE_CMDTYPE_ERR_INT_EN | \ + B_BE_PLE_QUE_DSTQUEID_ERR_INT_EN | \ + B_BE_PLE_QUE_SRCQUEID_ERR_INT_EN | \ + B_BE_PLE_ENQ_PKTCNT_OVRF_ERR_INT_EN | \ + B_BE_PLE_ENQ_PKTCNT_NVAL_ERR_INT_EN | \ + B_BE_PLE_PREPKTLLT_AD_ERR_INT_EN | \ + B_BE_PLE_NXTPKTLL_AD_ERR_INT_EN | \ + B_BE_PLE_QUEMGN_FRZTO_ERR_INT_EN | \ + B_BE_PLE_DATCHN_ARBT_ERR_INT_EN | \ + B_BE_PLE_DATCHN_NULLPG_ERR_INT_EN | \ + B_BE_PLE_DATCHN_FRZTO_ERR_INT_EN | \ + B_BE_PLE_DATCHN_RRDY_ERR_INT_EN | \ + B_BE_PLE_DATCHN_ADRERR_ERR_INT_EN | \ + B_BE_PLE_DATCHN_CAMREQ_ERR_INT_EN) + +#define R_BE_PLE_QTA0_CFG 0x9040 +#define B_BE_PLE_Q0_MAX_SIZE_MASK GENMASK(27, 16) +#define B_BE_PLE_Q0_MIN_SIZE_MASK GENMASK(11, 0) + +#define R_BE_PLE_QTA1_CFG 0x9044 +#define B_BE_PLE_Q1_MAX_SIZE_MASK GENMASK(27, 16) +#define B_BE_PLE_Q1_MIN_SIZE_MASK GENMASK(11, 0) + +#define R_BE_PLE_QTA2_CFG 0x9048 +#define B_BE_PLE_Q2_MAX_SIZE_MASK GENMASK(27, 16) +#define B_BE_PLE_Q2_MIN_SIZE_MASK GENMASK(11, 0) + +#define R_BE_PLE_QTA3_CFG 0x904C +#define B_BE_PLE_Q3_MAX_SIZE_MASK GENMASK(27, 16) +#define B_BE_PLE_Q3_MIN_SIZE_MASK GENMASK(11, 0) + +#define R_BE_PLE_QTA4_CFG 0x9050 +#define B_BE_PLE_Q4_MAX_SIZE_MASK GENMASK(27, 16) +#define B_BE_PLE_Q4_MIN_SIZE_MASK GENMASK(11, 0) + +#define R_BE_PLE_QTA5_CFG 0x9054 +#define B_BE_PLE_Q5_MAX_SIZE_MASK GENMASK(27, 16) +#define B_BE_PLE_Q5_MIN_SIZE_MASK GENMASK(11, 0) + +#define R_BE_PLE_QTA6_CFG 0x9058 +#define B_BE_PLE_Q6_MAX_SIZE_MASK GENMASK(27, 16) +#define B_BE_PLE_Q6_MIN_SIZE_MASK GENMASK(11, 0) + +#define R_BE_PLE_QTA7_CFG 0x905C +#define B_BE_PLE_Q7_MAX_SIZE_MASK GENMASK(27, 16) +#define B_BE_PLE_Q7_MIN_SIZE_MASK GENMASK(11, 0) + +#define R_BE_PLE_QTA8_CFG 0x9060 +#define B_BE_PLE_Q8_MAX_SIZE_MASK GENMASK(27, 16) +#define B_BE_PLE_Q8_MIN_SIZE_MASK GENMASK(11, 0) + +#define R_BE_PLE_QTA9_CFG 0x9064 +#define B_BE_PLE_Q9_MAX_SIZE_MASK GENMASK(27, 16) +#define B_BE_PLE_Q9_MIN_SIZE_MASK GENMASK(11, 0) + +#define R_BE_PLE_QTA10_CFG 0x9068 +#define B_BE_PLE_Q10_MAX_SIZE_MASK GENMASK(27, 16) +#define B_BE_PLE_Q10_MIN_SIZE_MASK GENMASK(11, 0) + +#define R_BE_PLE_QTA11_CFG 0x906C +#define B_BE_PLE_Q11_MAX_SIZE_MASK GENMASK(27, 16) +#define B_BE_PLE_Q11_MIN_SIZE_MASK GENMASK(11, 0) + +#define R_BE_PLE_QTA12_CFG 0x9070 +#define B_BE_PLE_Q12_MAX_SIZE_MASK GENMASK(27, 16) +#define B_BE_PLE_Q12_MIN_SIZE_MASK GENMASK(11, 0) + +#define R_BE_PLE_ERRFLAG1_IMR 0x90C0 +#define B_BE_PLE_SRCHPG_PGOFST_IMR BIT(26) +#define B_BE_PLE_SRCHPG_STRPG_IMR BIT(25) +#define B_BE_PLE_SRCHPG_FRZTO_IMR BIT(24) +#define B_BE_PLE_ERRFLAG1_IMR_CLR (B_BE_PLE_SRCHPG_FRZTO_IMR | \ + B_BE_PLE_SRCHPG_STRPG_IMR | \ + B_BE_PLE_SRCHPG_PGOFST_IMR) +#define B_BE_PLE_ERRFLAG1_IMR_SET (B_BE_PLE_SRCHPG_FRZTO_IMR | \ + B_BE_PLE_SRCHPG_STRPG_IMR | \ + B_BE_PLE_SRCHPG_PGOFST_IMR) + #define R_BE_PLE_DBG_FUN_INTF_CTL 0x9110 #define B_BE_PLE_DFI_ACTIVE BIT(31) #define B_BE_PLE_DFI_TRGSEL_MASK GENMASK(19, 16) @@ -4152,6 +5202,381 @@ #define R_BE_PLE_DBG_FUN_INTF_DATA 0x9114 #define B_BE_PLE_DFI_DATA_MASK GENMASK(31, 0) +#define R_BE_WDRLS_CFG 0x9408 +#define B_BE_WDRLS_DIS_AGAC BIT(31) +#define B_BE_RLSRPT_BUFREQ_TO_MASK GENMASK(15, 8) +#define B_BE_RLSRPT_BUFREQ_TO_SEL_MASK GENMASK(7, 6) +#define B_BE_WDRLS_MODE_MASK GENMASK(1, 0) + +#define R_BE_WDRLS_ERR_IMR 0x9430 +#define B_BE_WDRLS_RPT3_FRZTO_ERR_INT_EN BIT(21) +#define B_BE_WDRLS_RPT3_AGGNUM0_ERR_INT_EN BIT(20) +#define B_BE_WDRLS_RPT2_FRZTO_ERR_INT_EN BIT(17) +#define B_BE_WDRLS_RPT2_AGGNUM0_ERR_INT_EN BIT(16) +#define B_BE_WDRLS_RPT1_FRZTO_ERR_INT_EN BIT(13) +#define B_BE_WDRLS_RPT1_AGGNUM0_ERR_INT_EN BIT(12) +#define B_BE_WDRLS_RPT0_FRZTO_ERR_INT_EN BIT(9) +#define B_BE_WDRLS_RPT0_AGGNUM0_ERR_INT_EN BIT(8) +#define B_BE_WDRLS_PLEBREQ_PKTID_ISNULL_ERR_INT_EN BIT(5) +#define B_BE_WDRLS_PLEBREQ_TO_ERR_INT_EN BIT(4) +#define B_BE_WDRLS_CTL_FRZTO_ERR_INT_EN BIT(2) +#define B_BE_WDRLS_CTL_PLPKTID_ISNULL_ERR_INT_EN BIT(1) +#define B_BE_WDRLS_CTL_WDPKTID_ISNULL_ERR_INT_EN BIT(0) +#define B_BE_WDRLS_ERR_IMR_CLR (B_BE_WDRLS_CTL_WDPKTID_ISNULL_ERR_INT_EN | \ + B_BE_WDRLS_CTL_PLPKTID_ISNULL_ERR_INT_EN | \ + B_BE_WDRLS_CTL_FRZTO_ERR_INT_EN | \ + B_BE_WDRLS_PLEBREQ_TO_ERR_INT_EN | \ + B_BE_WDRLS_PLEBREQ_PKTID_ISNULL_ERR_INT_EN | \ + B_BE_WDRLS_RPT0_AGGNUM0_ERR_INT_EN | \ + B_BE_WDRLS_RPT0_FRZTO_ERR_INT_EN | \ + B_BE_WDRLS_RPT1_AGGNUM0_ERR_INT_EN | \ + B_BE_WDRLS_RPT1_FRZTO_ERR_INT_EN) +#define B_BE_WDRLS_ERR_IMR_SET (B_BE_WDRLS_CTL_WDPKTID_ISNULL_ERR_INT_EN | \ + B_BE_WDRLS_CTL_PLPKTID_ISNULL_ERR_INT_EN | \ + B_BE_WDRLS_CTL_FRZTO_ERR_INT_EN | \ + B_BE_WDRLS_PLEBREQ_PKTID_ISNULL_ERR_INT_EN | \ + B_BE_WDRLS_RPT0_AGGNUM0_ERR_INT_EN | \ + B_BE_WDRLS_RPT0_FRZTO_ERR_INT_EN | \ + B_BE_WDRLS_RPT1_AGGNUM0_ERR_INT_EN | \ + B_BE_WDRLS_RPT1_FRZTO_ERR_INT_EN) + +#define R_BE_RLSRPT0_CFG1 0x9444 +#define B_BE_RLSRPT0_FLTR_MAP_MASK GENMASK(27, 24) +#define S_BE_WDRLS_FLTR_TXOK 1 +#define S_BE_WDRLS_FLTR_RTYLMT 2 +#define S_BE_WDRLS_FLTR_LIFTIM 4 +#define S_BE_WDRLS_FLTR_MACID 8 +#define B_BE_RLSRPT0_TO_MASK GENMASK(23, 16) +#define B_BE_RLSRPT0_AGGNUM_MASK GENMASK(7, 0) + +#define R_BE_BBRPT_COM_ERR_IMR 0x9608 +#define B_BE_BBRPT_COM_EVT01_ISR_EN BIT(1) +#define B_BE_BBRPT_COM_NULL_PLPKTID_ISR_EN BIT(0) +#define B_BE_BBRPT_COM_ERR_IMR_CLR (B_BE_BBRPT_COM_NULL_PLPKTID_ISR_EN | \ + B_BE_BBRPT_COM_EVT01_ISR_EN) +#define B_BE_BBRPT_COM_ERR_IMR_SET B_BE_BBRPT_COM_NULL_PLPKTID_ISR_EN + +#define R_BE_BBRPT_CHINFO_ERR_IMR 0x9628 +#define B_BE_ERR_BB_ONETEN_INT_EN BIT(1) +#define B_BE_ERR_GEN_FRZTO_INT_EN BIT(0) +#define B_BE_BBRPT_CHINFO_ERR_IMR_CLR (B_BE_ERR_GEN_FRZTO_INT_EN | \ + B_BE_ERR_BB_ONETEN_INT_EN) +#define B_BE_BBRPT_CHINFO_ERR_IMR_SET (B_BE_ERR_GEN_FRZTO_INT_EN | \ + B_BE_ERR_BB_ONETEN_INT_EN) + +#define R_BE_BBRPT_DFS_ERR_IMR 0x9638 +#define B_BE_BBRPT_DFS_TO_ERR_INT_EN BIT(0) +#define B_BE_BBRPT_DFS_ERR_IMR_CLR B_BE_BBRPT_DFS_TO_ERR_INT_EN +#define B_BE_BBRPT_DFS_ERR_IMR_SET B_BE_BBRPT_DFS_TO_ERR_INT_EN + +#define R_BE_LA_ERRFLAG_IMR 0x9668 +#define B_BE_LA_IMR_DATA_LOSS BIT(0) +#define B_BE_LA_ERRFLAG_IMR_CLR B_BE_LA_IMR_DATA_LOSS +#define B_BE_LA_ERRFLAG_IMR_SET B_BE_LA_IMR_DATA_LOSS + +#define R_BE_LA_ERRFLAG_ISR 0x966C +#define B_BE_LA_ISR_DATA_LOSS BIT(0) + +#define R_BE_CH_INFO_DBGFLAG_IMR 0x9688 +#define B_BE_BCHN_EVT01_ISR_EN BIT(29) +#define B_BE_BCHN_REQTO_ISR_EN BIT(28) +#define B_BE_CHIF_RXDATA_AFACT_ISR_EN BIT(11) +#define B_BE_CHIF_RXDATA_BFACT_ISR_EN BIT(10) +#define B_BE_CHIF_HDR_SEGLEN_ISR_EN BIT(9) +#define B_BE_CHIF_HDR_INVLD_ISR_EN BIT(8) +#define B_BE_CHIF_BBONL_BFACT_ISR_EN BIT(4) +#define B_BE_CHIF_RPT_OVF_ISR_EN BIT(3) +#define B_BE_DBG_CHIF_DATA_LOSS_ISR_EN BIT(2) +#define B_BE_CHIF_DATA_WTOUT_ISR_EN BIT(1) +#define B_BE_CHIF_RPT_WTOUT_ISR_EN BIT(0) +#define B_BE_CH_INFO_DBGFLAG_IMR_CLR (B_BE_CHIF_RPT_WTOUT_ISR_EN | \ + B_BE_CHIF_DATA_WTOUT_ISR_EN | \ + B_BE_DBG_CHIF_DATA_LOSS_ISR_EN | \ + B_BE_CHIF_RPT_OVF_ISR_EN | \ + B_BE_CHIF_HDR_INVLD_ISR_EN | \ + B_BE_CHIF_HDR_SEGLEN_ISR_EN | \ + B_BE_CHIF_RXDATA_BFACT_ISR_EN | \ + B_BE_CHIF_RXDATA_AFACT_ISR_EN) +#define B_BE_CH_INFO_DBGFLAG_IMR_SET 0 + +#define R_BE_WD_BUF_REQ 0x9800 +#define B_BE_WD_BUF_REQ_EXEC BIT(31) +#define B_BE_WD_BUF_REQ_QUOTA_ID_MASK GENMASK(23, 16) +#define B_BE_WD_BUF_REQ_LEN_MASK GENMASK(15, 0) + +#define R_BE_WD_BUF_STATUS 0x9804 +#define B_BE_WD_BUF_STAT_DONE BIT(31) +#define B_BE_WD_BUF_STAT_PKTID_MASK GENMASK(11, 0) + +#define R_BE_WD_CPUQ_OP_0 0x9810 +#define B_BE_WD_CPUQ_OP_EXEC BIT(31) +#define B_BE_WD_CPUQ_OP_CMD_TYPE_MASK GENMASK(27, 24) +#define B_BE_WD_CPUQ_OP_PKTNUM_MASK GENMASK(7, 0) + +#define R_BE_WD_CPUQ_OP_1 0x9814 +#define B_BE_WD_CPUQ_OP_SRC_MACID_MASK GENMASK(19, 12) +#define B_BE_WD_CPUQ_OP_SRC_QID_MASK GENMASK(9, 4) +#define B_BE_WD_CPUQ_OP_SRC_PID_MASK GENMASK(2, 0) + +#define R_BE_WD_CPUQ_OP_2 0x9818 +#define B_BE_WD_CPUQ_OP_DST_MACID_MASK GENMASK(19, 12) +#define B_BE_WD_CPUQ_OP_DST_QID_MASK GENMASK(9, 4) +#define B_BE_WD_CPUQ_OP_DST_PID_MASK GENMASK(2, 0) + +#define R_BE_WD_CPUQ_OP_3 0x981C +#define B_BE_WD_CPUQ_OP_STRT_PKTID_MASK GENMASK(27, 16) +#define B_BE_WD_CPUQ_OP_END_PKTID_MASK GENMASK(11, 0) + +#define R_BE_WD_CPUQ_OP_STATUS 0x9820 +#define B_BE_WD_CPUQ_OP_STAT_DONE BIT(31) +#define B_BE_WD_CPUQ_OP_PKTCNT_MASK GENMASK(27, 16) +#define B_BE_WD_CPUQ_OP_PKTID_MASK GENMASK(11, 0) + +#define R_BE_PL_BUF_REQ 0x9840 +#define B_BE_PL_BUF_REQ_EXEC BIT(31) +#define B_BE_PL_BUF_REQ_QUOTA_ID_MASK GENMASK(19, 16) +#define B_BE_PL_BUF_REQ_LEN_MASK GENMASK(15, 0) + +#define R_BE_PL_BUF_STATUS 0x9844 +#define B_BE_PL_BUF_STAT_DONE BIT(31) +#define B_BE_PL_BUF_STAT_PKTID_MASK GENMASK(11, 0) + +#define R_BE_PL_CPUQ_OP_0 0x9850 +#define B_BE_PL_CPUQ_OP_EXEC BIT(31) +#define B_BE_PL_CPUQ_OP_CMD_TYPE_MASK GENMASK(27, 24) +#define B_BE_PL_CPUQ_OP_PKTNUM_MASK GENMASK(7, 0) + +#define R_BE_PL_CPUQ_OP_1 0x9854 +#define B_BE_PL_CPUQ_OP_SRC_MACID_MASK GENMASK(19, 12) +#define B_BE_PL_CPUQ_OP_SRC_QID_MASK GENMASK(9, 4) +#define B_BE_PL_CPUQ_OP_SRC_PID_MASK GENMASK(2, 0) + +#define R_BE_PL_CPUQ_OP_2 0x9858 +#define B_BE_PL_CPUQ_OP_DST_MACID_MASK GENMASK(19, 12) +#define B_BE_PL_CPUQ_OP_DST_QID_MASK GENMASK(9, 4) +#define B_BE_PL_CPUQ_OP_DST_PID_MASK GENMASK(2, 0) + +#define R_BE_PL_CPUQ_OP_3 0x985C +#define B_BE_PL_CPUQ_OP_STRT_PKTID_MASK GENMASK(27, 16) +#define B_BE_PL_CPUQ_OP_END_PKTID_MASK GENMASK(11, 0) + +#define R_BE_PL_CPUQ_OP_STATUS 0x9860 +#define B_BE_PL_CPUQ_OP_STAT_DONE BIT(31) +#define B_BE_PL_CPUQ_OP_PKTCNT_MASK GENMASK(27, 16) +#define B_BE_PL_CPUQ_OP_PKTID_MASK GENMASK(11, 0) + +#define R_BE_CPUIO_ERR_IMR 0x9888 +#define B_BE_PLEQUE_OP_ERR_INT_EN BIT(12) +#define B_BE_PLEBUF_OP_ERR_INT_EN BIT(8) +#define B_BE_WDEQUE_OP_ERR_INT_EN BIT(4) +#define B_BE_WDEBUF_OP_ERR_INT_EN BIT(0) +#define B_BE_CPUIO_ERR_IMR_CLR (B_BE_WDEBUF_OP_ERR_INT_EN | \ + B_BE_WDEQUE_OP_ERR_INT_EN | \ + B_BE_PLEBUF_OP_ERR_INT_EN | \ + B_BE_PLEQUE_OP_ERR_INT_EN) +#define B_BE_CPUIO_ERR_IMR_SET (B_BE_WDEBUF_OP_ERR_INT_EN | \ + B_BE_WDEQUE_OP_ERR_INT_EN | \ + B_BE_PLEBUF_OP_ERR_INT_EN | \ + B_BE_PLEQUE_OP_ERR_INT_EN) + +#define R_BE_PKTIN_ERR_IMR 0x9A20 +#define B_BE_SW_MERGE_ERR_INT_EN BIT(1) +#define B_BE_GET_NULL_PKTID_ERR_INT_EN BIT(0) +#define B_BE_PKTIN_ERR_IMR_CLR (B_BE_SW_MERGE_ERR_INT_EN | \ + B_BE_GET_NULL_PKTID_ERR_INT_EN) +#define B_BE_PKTIN_ERR_IMR_SET (B_BE_SW_MERGE_ERR_INT_EN | \ + B_BE_GET_NULL_PKTID_ERR_INT_EN) + +#define R_BE_MPDU_TX_ERR_IMR 0x9BF4 +#define B_BE_TX_TIMEOUT_ERR_EN BIT(0) +#define B_BE_MPDU_TX_ERR_IMR_CLR B_BE_TX_TIMEOUT_ERR_EN +#define B_BE_MPDU_TX_ERR_IMR_SET 0 + +#define R_BE_MPDU_RX_ERR_IMR 0x9CF4 +#define B_BE_LEN_ERR_IMR BIT(3) +#define B_BE_TIMEOUT_ERR_IMR BIT(1) +#define B_BE_MPDU_RX_ERR_IMR_CLR B_BE_TIMEOUT_ERR_IMR +#define B_BE_MPDU_RX_ERR_IMR_SET 0 + +#define R_BE_SEC_ENG_CTRL 0x9D00 +#define B_BE_SEC_ENG_EN BIT(31) +#define B_BE_CCMP_SPP_MIC BIT(30) +#define B_BE_CCMP_SPP_CTR BIT(29) +#define B_BE_SEC_CAM_ACC BIT(28) +#define B_BE_WMAC_SEC_PN_SEL_MASK GENMASK(27, 26) +#define B_BE_WMAC_SEC_MASKIV BIT(25) +#define B_BE_WAPI_SPEC BIT(24) +#define B_BE_REVERT_TA_RA_MLD_EN BIT(23) +#define B_BE_SEC_DBG_SEL_MASK GENMASK(19, 16) +#define B_BE_CAM_FORCE_CLK BIT(15) +#define B_BE_SEC_FORCE_CLK BIT(14) +#define B_BE_SEC_RX_SHORT_ADD_ICVERR BIT(13) +#define B_BE_SRAM_IO_PROT BIT(12) +#define B_BE_SEC_PRE_ENQUE_TX BIT(11) +#define B_BE_CLK_EN_CGCMP BIT(10) +#define B_BE_CLK_EN_WAPI BIT(9) +#define B_BE_CLK_EN_WEP_TKIP BIT(8) +#define B_BE_BMC_MGNT_DEC BIT(5) +#define B_BE_UC_MGNT_DEC BIT(4) +#define B_BE_MC_DEC BIT(3) +#define B_BE_BC_DEC BIT(2) +#define B_BE_SEC_RX_DEC BIT(1) +#define B_BE_SEC_TX_ENC BIT(0) + +#define R_BE_SEC_MPDU_PROC 0x9D04 +#define B_BE_DBG_ENGINE_SEL BIT(8) +#define B_BE_STOP_RX_PKT_HANDLE BIT(7) +#define B_BE_STOP_TX_PKT_HANDLE BIT(6) +#define B_BE_QUEUE_FOWARD_SEL BIT(5) +#define B_BE_RESP1_PROTECT BIT(4) +#define B_BE_RESP0_PROTECT BIT(3) +#define B_BE_TX_ACTIVE_PROTECT BIT(2) +#define B_BE_APPEND_ICV BIT(1) +#define B_BE_APPEND_MIC BIT(0) + +#define R_BE_SEC_CAM_ACCESS 0x9D10 +#define B_BE_SEC_TIME_OUT_MASK GENMASK(31, 16) +#define B_BE_SEC_CAM_POLL BIT(15) +#define B_BE_SEC_CAM_RW BIT(14) +#define B_BE_SEC_CAM_ACC_FAIL BIT(13) +#define B_BE_SEC_CAM_OFFSET_MASK GENMASK(10, 0) + +#define R_BE_SEC_CAM_RDATA 0x9D14 +#define B_BE_SEC_CAM_RDATA_MASK GENMASK(31, 0) + +#define R_BE_SEC_DEBUG2 0x9D28 +#define B_BE_DBG_READ_MASK GENMASK(31, 0) + +#define R_BE_SEC_ERROR_IMR 0x9D2C +#define B_BE_QUEUE_OPERATION_HANG_IMR BIT(4) +#define B_BE_SEC1_RX_HANG_IMR BIT(3) +#define B_BE_SEC1_TX_HANG_IMR BIT(2) +#define B_BE_RX_HANG_IMR BIT(1) +#define B_BE_TX_HANG_IMR BIT(0) +#define B_BE_SEC_ERROR_IMR_CLR (B_BE_TX_HANG_IMR | \ + B_BE_RX_HANG_IMR | \ + B_BE_SEC1_TX_HANG_IMR | \ + B_BE_SEC1_RX_HANG_IMR | \ + B_BE_QUEUE_OPERATION_HANG_IMR) +#define B_BE_SEC_ERROR_IMR_SET (B_BE_TX_HANG_IMR | \ + B_BE_RX_HANG_IMR | \ + B_BE_SEC1_TX_HANG_IMR | \ + B_BE_SEC1_RX_HANG_IMR | \ + B_BE_QUEUE_OPERATION_HANG_IMR) + +#define R_BE_SEC_ERROR_FLAG 0x9D30 +#define B_BE_TXD_DIFF_KEYCAM_TYPE_ERROR BIT(5) +#define B_BE_QUEUE_OPERATION_HANG_ERROR BIT(4) +#define B_BE_SEC1_RX_HANG_ERROR BIT(3) +#define B_BE_SEC1_TX_HANG_ERROR BIT(2) +#define B_BE_RX_HANG_ERROR BIT(1) +#define B_BE_TX_HANG_ERROR BIT(0) + +#define R_BE_TXPKTCTL_B0_ERRFLAG_IMR 0x9F78 +#define B_BE_B0_IMR_DBG_USRCTL_RLSBMPLEN BIT(25) +#define B_BE_B0_IMR_DBG_USRCTL_RDNRLSCMD BIT(24) +#define B_BE_B0_IMR_ERR_PRELD_ENTNUMCFG BIT(17) +#define B_BE_B0_IMR_ERR_PRELD_RLSPKTSZERR BIT(16) +#define B_BE_B0_IMR_ERR_CMDPSR_TBLSZ BIT(11) +#define B_BE_B0_IMR_ERR_CMDPSR_FRZTO BIT(10) +#define B_BE_B0_IMR_ERR_CMDPSR_CMDTYPE BIT(9) +#define B_BE_B0_IMR_ERR_CMDPSR_1STCMDERR BIT(8) +#define B_BE_B0_IMR_ERR_USRCTL_NOINIT BIT(1) +#define B_BE_B0_IMR_ERR_USRCTL_REINIT BIT(0) +#define B_BE_TXPKTCTL_B0_ERRFLAG_IMR_CLR (B_BE_B0_IMR_ERR_USRCTL_REINIT | \ + B_BE_B0_IMR_ERR_USRCTL_NOINIT | \ + B_BE_B0_IMR_DBG_USRCTL_RDNRLSCMD | \ + B_BE_B0_IMR_DBG_USRCTL_RLSBMPLEN | \ + B_BE_B0_IMR_ERR_CMDPSR_1STCMDERR | \ + B_BE_B0_IMR_ERR_CMDPSR_CMDTYPE | \ + B_BE_B0_IMR_ERR_CMDPSR_FRZTO | \ + B_BE_B0_IMR_ERR_CMDPSR_TBLSZ | \ + B_BE_B0_IMR_ERR_PRELD_RLSPKTSZERR | \ + B_BE_B0_IMR_ERR_PRELD_ENTNUMCFG) +#define B_BE_TXPKTCTL_B0_ERRFLAG_IMR_SET (B_BE_B0_IMR_ERR_USRCTL_REINIT | \ + B_BE_B0_IMR_ERR_USRCTL_NOINIT | \ + B_BE_B0_IMR_ERR_CMDPSR_1STCMDERR | \ + B_BE_B0_IMR_ERR_CMDPSR_CMDTYPE | \ + B_BE_B0_IMR_ERR_CMDPSR_FRZTO | \ + B_BE_B0_IMR_ERR_CMDPSR_TBLSZ | \ + B_BE_B0_IMR_ERR_PRELD_RLSPKTSZERR | \ + B_BE_B0_IMR_ERR_PRELD_ENTNUMCFG) + +#define R_BE_TXPKTCTL_B1_ERRFLAG_IMR 0x9FB8 +#define B_BE_B1_IMR_DBG_USRCTL_RLSBMPLEN BIT(25) +#define B_BE_B1_IMR_DBG_USRCTL_RDNRLSCMD BIT(24) +#define B_BE_B1_IMR_ERR_PRELD_ENTNUMCFG BIT(17) +#define B_BE_B1_IMR_ERR_PRELD_RLSPKTSZERR BIT(16) +#define B_BE_B1_IMR_ERR_CMDPSR_TBLSZ BIT(11) +#define B_BE_B1_IMR_ERR_CMDPSR_FRZTO BIT(10) +#define B_BE_B1_IMR_ERR_CMDPSR_CMDTYPE BIT(9) +#define B_BE_B1_IMR_ERR_CMDPSR_1STCMDERR BIT(8) +#define B_BE_B1_IMR_ERR_USRCTL_NOINIT BIT(1) +#define B_BE_B1_IMR_ERR_USRCTL_REINIT BIT(0) +#define B_BE_TXPKTCTL_B1_ERRFLAG_IMR_CLR (B_BE_B1_IMR_ERR_USRCTL_REINIT | \ + B_BE_B1_IMR_ERR_USRCTL_NOINIT | \ + B_BE_B1_IMR_DBG_USRCTL_RDNRLSCMD | \ + B_BE_B1_IMR_DBG_USRCTL_RLSBMPLEN | \ + B_BE_B1_IMR_ERR_CMDPSR_1STCMDERR | \ + B_BE_B1_IMR_ERR_CMDPSR_CMDTYPE | \ + B_BE_B1_IMR_ERR_CMDPSR_FRZTO | \ + B_BE_B1_IMR_ERR_CMDPSR_TBLSZ | \ + B_BE_B1_IMR_ERR_PRELD_RLSPKTSZERR | \ + B_BE_B1_IMR_ERR_PRELD_ENTNUMCFG) +#define B_BE_TXPKTCTL_B1_ERRFLAG_IMR_SET (B_BE_B1_IMR_ERR_USRCTL_REINIT | \ + B_BE_B1_IMR_ERR_USRCTL_NOINIT | \ + B_BE_B1_IMR_ERR_CMDPSR_1STCMDERR | \ + B_BE_B1_IMR_ERR_CMDPSR_CMDTYPE | \ + B_BE_B1_IMR_ERR_CMDPSR_FRZTO | \ + B_BE_B1_IMR_ERR_CMDPSR_TBLSZ | \ + B_BE_B1_IMR_ERR_PRELD_RLSPKTSZERR | \ + B_BE_B1_IMR_ERR_PRELD_ENTNUMCFG) + +#define R_BE_MLO_ERR_IDCT_IMR 0xA128 +#define B_BE_MLO_ERR_IDCT_IMR_0 BIT(31) +#define B_BE_MLO_ERR_IDCT_IMR_1 BIT(30) +#define B_BE_MLO_ERR_IDCT_IMR_2 BIT(29) +#define B_BE_MLO_ERR_IDCT_IMR_3 BIT(28) +#define B_BE_MLO_ERR_IDCT_IMR_CLR (B_BE_MLO_ERR_IDCT_IMR_2 | \ + B_BE_MLO_ERR_IDCT_IMR_1 | \ + B_BE_MLO_ERR_IDCT_IMR_0) +#define B_BE_MLO_ERR_IDCT_IMR_SET (B_BE_MLO_ERR_IDCT_IMR_2 | \ + B_BE_MLO_ERR_IDCT_IMR_1 | \ + B_BE_MLO_ERR_IDCT_IMR_0) + +#define R_BE_MLO_ERR_IDCT_ISR 0xA12C +#define B_BE_MLO_ISR_IDCT_0 BIT(31) +#define B_BE_MLO_ISR_IDCT_1 BIT(30) +#define B_BE_MLO_ISR_IDCT_2 BIT(29) +#define B_BE_MLO_ISR_IDCT_3 BIT(28) + +#define R_BE_PLRLS_ERR_IMR 0xA218 +#define B_BE_PLRLS_CTL_FRZTO_IMR BIT(0) +#define B_BE_PLRLS_ERR_IMR_CLR B_BE_PLRLS_CTL_FRZTO_IMR +#define B_BE_PLRLS_ERR_IMR_SET B_BE_PLRLS_CTL_FRZTO_IMR + +#define R_BE_PLRLS_ERR_ISR 0xA21C +#define B_BE_PLRLS_CTL_EVT03_ISR BIT(3) +#define B_BE_PLRLS_CTL_EVT02_ISR BIT(2) +#define B_BE_PLRLS_CTL_EVT01_ISR BIT(1) +#define B_BE_PLRLS_CTL_FRZTO_ISR BIT(0) + +#define R_BE_INTERRUPT_MASK_REG 0xA3F0 +#define B_BE_PLE_B_PKTID_ERR_IMR BIT(2) +#define B_BE_RPT_TIMEOUT_IMR BIT(1) +#define B_BE_SEARCH_TIMEOUT_IMR BIT(0) +#define B_BE_INTERRUPT_MASK_REG_CLR (B_BE_SEARCH_TIMEOUT_IMR | \ + B_BE_RPT_TIMEOUT_IMR | \ + B_BE_PLE_B_PKTID_ERR_IMR) +#define B_BE_INTERRUPT_MASK_REG_SET (B_BE_SEARCH_TIMEOUT_IMR | \ + B_BE_RPT_TIMEOUT_IMR | \ + B_BE_PLE_B_PKTID_ERR_IMR) + +#define R_BE_INTERRUPT_STS_REG 0xA3F4 +#define B_BE_PLE_B_PKTID_ERR_ISR BIT(2) +#define B_BE_RPT_TIMEOUT_ISR BIT(1) +#define B_BE_SEARCH_TIMEOUT_ISR BIT(0) + #define R_BE_HAXI_INIT_CFG1 0xB000 #define B_BE_CFG_WD_PERIOD_IDLE_MASK GENMASK(31, 28) #define B_BE_CFG_WD_PERIOD_ACTIVE_MASK GENMASK(27, 24) @@ -4174,6 +5599,112 @@ #define B_BE_MAX_RXDMA_MASK GENMASK(3, 2) #define B_BE_MAX_TXDMA_MASK GENMASK(1, 0) +#define R_BE_HAXI_DMA_STOP1 0xB010 +#define B_BE_STOP_WPDMA BIT(31) +#define B_BE_STOP_CH14 BIT(14) +#define B_BE_STOP_CH13 BIT(13) +#define B_BE_STOP_CH12 BIT(12) +#define B_BE_STOP_CH11 BIT(11) +#define B_BE_STOP_CH10 BIT(10) +#define B_BE_STOP_CH9 BIT(9) +#define B_BE_STOP_CH8 BIT(8) +#define B_BE_STOP_CH7 BIT(7) +#define B_BE_STOP_CH6 BIT(6) +#define B_BE_STOP_CH5 BIT(5) +#define B_BE_STOP_CH4 BIT(4) +#define B_BE_STOP_CH3 BIT(3) +#define B_BE_STOP_CH2 BIT(2) +#define B_BE_STOP_CH1 BIT(1) +#define B_BE_STOP_CH0 BIT(0) + +#define R_BE_HAXI_IDCT_MSK 0xB0B8 +#define B_BE_HAXI_RRESP_ERR_IDCT_MSK BIT(7) +#define B_BE_HAXI_BRESP_ERR_IDCT_MSK BIT(6) +#define B_BE_RXDMA_ERR_FLAG_IDCT_MSK BIT(5) +#define B_BE_SET_FC_ERROR_FLAG_IDCT_MSK BIT(4) +#define B_BE_TXBD_LEN0_ERR_IDCT_MSK BIT(3) +#define B_BE_TXBD_4KBOUND_ERR_IDCT_MSK BIT(2) +#define B_BE_RXMDA_STUCK_IDCT_MSK BIT(1) +#define B_BE_TXMDA_STUCK_IDCT_MSK BIT(0) +#define B_BE_HAXI_IDCT_MSK_CLR (B_BE_TXMDA_STUCK_IDCT_MSK | \ + B_BE_RXMDA_STUCK_IDCT_MSK | \ + B_BE_TXBD_LEN0_ERR_IDCT_MSK | \ + B_BE_SET_FC_ERROR_FLAG_IDCT_MSK | \ + B_BE_RXDMA_ERR_FLAG_IDCT_MSK | \ + B_BE_HAXI_BRESP_ERR_IDCT_MSK | \ + B_BE_HAXI_RRESP_ERR_IDCT_MSK) +#define B_BE_HAXI_IDCT_MSK_SET (B_BE_TXMDA_STUCK_IDCT_MSK | \ + B_BE_RXMDA_STUCK_IDCT_MSK | \ + B_BE_TXBD_LEN0_ERR_IDCT_MSK | \ + B_BE_SET_FC_ERROR_FLAG_IDCT_MSK | \ + B_BE_RXDMA_ERR_FLAG_IDCT_MSK | \ + B_BE_HAXI_BRESP_ERR_IDCT_MSK | \ + B_BE_HAXI_RRESP_ERR_IDCT_MSK) + +#define R_BE_HAXI_IDCT 0xB0BC +#define B_BE_HAXI_RRESP_ERR_IDCT BIT(7) +#define B_BE_HAXI_BRESP_ERR_IDCT BIT(6) +#define B_BE_RXDMA_ERR_FLAG_IDCT BIT(5) +#define B_BE_SET_FC_ERROR_FLAG_IDCT BIT(4) +#define B_BE__TXBD_LEN0_ERR_IDCT BIT(3) +#define B_BE__TXBD_4KBOUND_ERR_IDCT BIT(2) +#define B_BE_RXMDA_STUCK_IDCT BIT(1) +#define B_BE_TXMDA_STUCK_IDCT BIT(0) + +#define R_BE_HCI_FC_CTRL 0xB700 +#define B_BE_WD_PAGE_MODE_MASK GENMASK(17, 16) +#define B_BE_HCI_FC_CH14_FULL_COND_MASK GENMASK(15, 14) +#define B_BE_HCI_FC_TWD_FULL_COND_MASK GENMASK(13, 12) +#define B_BE_HCI_FC_CH12_FULL_COND_MASK GENMASK(11, 10) +#define B_BE_HCI_FC_WP_CH811_FULL_COND_MASK GENMASK(9, 8) +#define B_BE_HCI_FC_WP_CH07_FULL_COND_MASK GENMASK(7, 6) +#define B_BE_HCI_FC_WD_FULL_COND_MASK GENMASK(5, 4) +#define B_BE_HCI_FC_CH12_EN BIT(3) +#define B_BE_HCI_FC_MODE_MASK GENMASK(2, 1) +#define B_BE_HCI_FC_EN BIT(0) + +#define R_BE_CH_PAGE_CTRL 0xB704 +#define B_BE_PREC_PAGE_CH12_V1_MASK GENMASK(21, 16) +#define B_BE_PREC_PAGE_CH011_V1_MASK GENMASK(5, 0) + +#define R_BE_PUB_PAGE_INFO3 0xB78C +#define B_BE_G1_AVAL_PG_MASK GENMASK(28, 16) +#define B_BE_G0_AVAL_PG_MASK GENMASK(12, 0) + +#define R_BE_PUB_PAGE_CTRL1 0xB790 +#define B_BE_PUBPG_G1_MASK GENMASK(28, 16) +#define B_BE_PUBPG_G0_MASK GENMASK(12, 0) + +#define R_BE_PUB_PAGE_CTRL2 0xB794 +#define B_BE_PUBPG_ALL_MASK GENMASK(12, 0) + +#define R_BE_PUB_PAGE_INFO1 0xB79C +#define B_BE_G1_USE_PG_MASK GENMASK(28, 16) +#define B_BE_G0_USE_PG_MASK GENMASK(12, 0) + +#define R_BE_PUB_PAGE_INFO2 0xB7A0 +#define B_BE_PUB_AVAL_PG_MASK GENMASK(12, 0) + +#define R_BE_WP_PAGE_CTRL1 0xB7A4 +#define B_BE_PREC_PAGE_WP_CH811_MASK GENMASK(24, 16) +#define B_BE_PREC_PAGE_WP_CH07_MASK GENMASK(8, 0) + +#define R_BE_WP_PAGE_CTRL2 0xB7A8 +#define B_BE_WP_THRD_MASK GENMASK(12, 0) + +#define R_BE_WP_PAGE_INFO1 0xB7AC +#define B_BE_WP_AVAL_PG_MASK GENMASK(28, 16) + +#define R_BE_CMAC_SHARE_FUNC_EN 0x0E000 +#define B_BE_CMAC_SHARE_CRPRT BIT(31) +#define B_BE_CMAC_SHARE_EN BIT(30) +#define B_BE_FORCE_BTCOEX_REG_GCKEN BIT(24) +#define B_BE_FORCE_CMAC_SHARE_COMMON_REG_GCKEN BIT(16) +#define B_BE_FORCE_CMAC_SHARE_REG_GCKEN BIT(15) +#define B_BE_RESPBA_EN BIT(2) +#define B_BE_ADDRSRCH_EN BIT(1) +#define B_BE_BTCOEX_EN BIT(0) + #define R_BE_CMAC_FUNC_EN 0x10000 #define R_BE_CMAC_FUNC_EN_C1 0x14000 #define B_BE_CMAC_CRPRT BIT(31) @@ -4206,6 +5737,104 @@ B_BE_CMAC_CRPRT | B_BE_TXTIME_EN | B_BE_RESP_PKTCTL_EN | \ B_BE_SIGB_EN) +#define R_BE_CK_EN 0x10004 +#define R_BE_CK_EN_C1 0x14004 +#define B_BE_CMAC_CKEN BIT(30) +#define B_BE_BCN_P1_P4_CKEN BIT(15) +#define B_BE_BCN_P0MB1_15_CKEN BIT(14) +#define B_BE_TXTIME_CKEN BIT(8) +#define B_BE_RESP_PKTCTL_CKEN BIT(7) +#define B_BE_SIGB_CKEN BIT(6) +#define B_BE_PHYINTF_CKEN BIT(5) +#define B_BE_CMAC_DMA_CKEN BIT(4) +#define B_BE_PTCLTOP_CKEN BIT(3) +#define B_BE_SCHEDULER_CKEN BIT(2) +#define B_BE_TMAC_CKEN BIT(1) +#define B_BE_RMAC_CKEN BIT(0) +#define B_BE_CK_EN_SET (B_BE_CMAC_CKEN | B_BE_PHYINTF_CKEN | B_BE_CMAC_DMA_CKEN | \ + B_BE_PTCLTOP_CKEN | B_BE_SCHEDULER_CKEN | B_BE_TMAC_CKEN | \ + B_BE_RMAC_CKEN | B_BE_TXTIME_CKEN | B_BE_RESP_PKTCTL_CKEN | \ + B_BE_SIGB_CKEN) + +#define R_BE_CMAC_ERR_IMR 0x10160 +#define R_BE_CMAC_ERR_IMR_C1 0x14160 +#define B_BE_CMAC_FW_ERR_IDCT_EN BIT(16) +#define B_BE_PTCL_TX_IDLETO_IDCT_EN BIT(9) +#define B_BE_WMAC_RX_IDLETO_IDCT_EN BIT(8) +#define B_BE_WMAC_TX_ERR_IND_EN BIT(7) +#define B_BE_WMAC_RX_ERR_IND_EN BIT(6) +#define B_BE_TXPWR_CTRL_ERR_IND_EN BIT(5) +#define B_BE_PHYINTF_ERR_IND_EN BIT(4) +#define B_BE_DMA_TOP_ERR_IND_EN BIT(3) +#define B_BE_RESP_PKTCTL_ERR_IND_EN BIT(2) +#define B_BE_PTCL_TOP_ERR_IND_EN BIT(1) +#define B_BE_SCHEDULE_TOP_ERR_IND_EN BIT(0) + +#define R_BE_CMAC_ERR_ISR 0x10164 +#define R_BE_CMAC_ERR_ISR_C1 0x14164 +#define B_BE_CMAC_FW_ERR_IDCT BIT(16) +#define B_BE_PTCL_TX_IDLETO_IDCT BIT(9) +#define B_BE_WMAC_RX_IDLETO_IDCT BIT(8) +#define B_BE_WMAC_TX_ERR_IND BIT(7) +#define B_BE_WMAC_RX_ERR_IND BIT(6) +#define B_BE_TXPWR_CTRL_ERR_IND BIT(5) +#define B_BE_PHYINTF_ERR_IND BIT(4) +#define B_BE_DMA_TOP_ERR_IND BIT(3) +#define B_BE_RESP_PKTCTL_ERR_IDCT BIT(2) +#define B_BE_PTCL_TOP_ERR_IND BIT(1) +#define B_BE_SCHEDULE_TOP_ERR_IND BIT(0) + +#define R_BE_SER_L0_DBG_CNT 0x10170 +#define R_BE_SER_L0_DBG_CNT_C1 0x14170 +#define B_BE_SER_L0_PHYINTF_CNT_MASK GENMASK(31, 24) +#define B_BE_SER_L0_DMA_CNT_MASK GENMASK(23, 16) +#define B_BE_SER_L0_PTCL_CNT_MASK GENMASK(15, 8) +#define B_BE_SER_L0_SCH_CNT_MASK GENMASK(7, 0) + +#define R_BE_SER_L0_DBG_CNT1 0x10174 +#define R_BE_SER_L0_DBG_CNT1_C1 0x14174 +#define B_BE_SER_L0_TMAC_COUNTER_MASK GENMASK(23, 16) +#define B_BE_SER_L0_RMAC_COUNTER_MASK GENMASK(15, 8) +#define B_BE_SER_L0_TXPWR_COUNTER_MASK GENMASK(7, 0) + +#define R_BE_SER_L0_DBG_CNT2 0x10178 +#define R_BE_SER_L0_DBG_CNT2_C1 0x14178 + +#define R_BE_SER_L0_DBG_CNT3 0x1017C +#define R_BE_SER_L0_DBG_CNT3_C1 0x1417C +#define B_BE_SER_L0_SUBMODULE_BIT31_CNT BIT(31) +#define B_BE_SER_L0_SUBMODULE_BIT30_CNT BIT(30) +#define B_BE_SER_L0_SUBMODULE_BIT29_CNT BIT(29) +#define B_BE_SER_L0_SUBMODULE_BIT28_CNT BIT(28) +#define B_BE_SER_L0_SUBMODULE_BIT27_CNT BIT(27) +#define B_BE_SER_L0_SUBMODULE_BIT26_CNT BIT(26) +#define B_BE_SER_L0_SUBMODULE_BIT25_CNT BIT(25) +#define B_BE_SER_L0_SUBMODULE_BIT24_CNT BIT(24) +#define B_BE_SER_L0_SUBMODULE_BIT23_CNT BIT(23) +#define B_BE_SER_L0_SUBMODULE_BIT22_CNT BIT(22) +#define B_BE_SER_L0_SUBMODULE_BIT21_CNT BIT(21) +#define B_BE_SER_L0_SUBMODULE_BIT20_CNT BIT(20) +#define B_BE_SER_L0_SUBMODULE_BIT19_CNT BIT(19) +#define B_BE_SER_L0_SUBMODULE_BIT18_CNT BIT(18) +#define B_BE_SER_L0_SUBMODULE_BIT17_CNT BIT(17) +#define B_BE_SER_L0_SUBMODULE_BIT16_CNT BIT(16) +#define B_BE_SER_L0_SUBMODULE_BIT15_CNT BIT(15) +#define B_BE_SER_L0_SUBMODULE_BIT14_CNT BIT(14) +#define B_BE_SER_L0_SUBMODULE_BIT13_CNT BIT(13) +#define B_BE_SER_L0_SUBMODULE_BIT12_CNT BIT(12) +#define B_BE_SER_L0_SUBMODULE_BIT11_CNT BIT(11) +#define B_BE_SER_L0_SUBMODULE_BIT10_CNT BIT(10) +#define B_BE_SER_L0_SUBMODULE_BIT9_CNT BIT(9) +#define B_BE_SER_L0_SUBMODULE_BIT8_CNT BIT(8) +#define B_BE_SER_L0_SUBMODULE_BIT7_CNT BIT(7) +#define B_BE_SER_L0_SUBMODULE_BIT6_CNT BIT(6) +#define B_BE_SER_L0_SUBMODULE_BIT5_CNT BIT(5) +#define B_BE_SER_L0_SUBMODULE_BIT4_CNT BIT(4) +#define B_BE_SER_L0_SUBMODULE_BIT3_CNT BIT(3) +#define B_BE_SER_L0_SUBMODULE_BIT2_CNT BIT(2) +#define B_BE_SER_L0_SUBMODULE_BIT1_CNT BIT(1) +#define B_BE_SER_L0_SUBMODULE_BIT0_CNT BIT(0) + #define R_BE_PORT_0_TSF_SYNC 0x102A0 #define R_BE_PORT_0_TSF_SYNC_C1 0x142A0 #define B_BE_P0_SYNC_NOW_P BIT(30) @@ -4226,6 +5855,17 @@ #define B_BE_SET_MUEDCATIMER_TF_0 BIT(4) #define B_BE_MUEDCA_EN_0 BIT(0) +#define R_BE_SCHEDULE_ERR_IMR 0x103E8 +#define R_BE_SCHEDULE_ERR_IMR_C1 0x143E8 +#define B_BE_FSM_TIMEOUT_ERR_INT_EN BIT(0) +#define B_BE_SCHEDULE_ERR_IMR_CLR B_BE_FSM_TIMEOUT_ERR_INT_EN +#define B_BE_SCHEDULE_ERR_IMR_SET B_BE_FSM_TIMEOUT_ERR_INT_EN + +#define R_BE_SCHEDULE_ERR_ISR 0x103EC +#define R_BE_SCHEDULE_ERR_ISR_C1 0x143EC +#define B_BE_SORT_NON_IDLE_ERR_INT BIT(1) +#define B_BE_FSM_TIMEOUT_ERR_INT BIT(0) + #define R_BE_PORT_CFG_P0 0x10400 #define R_BE_PORT_CFG_P0_C1 0x14400 #define B_BE_BCN_ERLY_SORT_EN_P0 BIT(18) @@ -4364,6 +6004,348 @@ #define R_BE_PTCL_BSS_COLOR_1_C1 0x148A4 #define B_BE_BSS_COLOB_BE_PORT_4_MASK GENMASK(5, 0) +#define R_BE_PTCL_IMR_2 0x108B8 +#define R_BE_PTCL_IMR_2_C1 0x148B8 +#define B_BE_NO_TRX_TIMEOUT_IMR BIT(1) +#define B_BE_TX_IDLE_TIMEOUT_IMR BIT(0) +#define B_BE_PTCL_IMR_2_CLR B_BE_TX_IDLE_TIMEOUT_IMR +#define B_BE_PTCL_IMR_2_SET 0 + +#define R_BE_PTCL_IMR0 0x108C0 +#define R_BE_PTCL_IMR0_C1 0x148C0 +#define B_BE_PTCL_ERROR_FLAG_IMR BIT(31) +#define B_BE_FSM1_TIMEOUT_ERR_INT_EN BIT(1) +#define B_BE_FSM_TIMEOUT_ERR_INT_EN BIT(0) +#define B_BE_PTCL_IMR0_CLR (B_BE_FSM_TIMEOUT_ERR_INT_EN | \ + B_BE_FSM1_TIMEOUT_ERR_INT_EN | \ + B_BE_PTCL_ERROR_FLAG_IMR) +#define B_BE_PTCL_IMR0_SET (B_BE_FSM_TIMEOUT_ERR_INT_EN | \ + B_BE_FSM1_TIMEOUT_ERR_INT_EN | \ + B_BE_PTCL_ERROR_FLAG_IMR) + +#define R_BE_PTCL_ISR0 0x108C4 +#define R_BE_PTCL_ISR0_C1 0x148C4 +#define B_BE_PTCL_ERROR_FLAG_ISR BIT(31) +#define B_BE_FSM1_TIMEOUT_ERR BIT(1) +#define B_BE_FSM_TIMEOUT_ERR BIT(0) + +#define R_BE_PTCL_IMR1 0x108C8 +#define R_BE_PTCL_IMR1_C1 0x148C8 +#define B_BE_F2PCMD_PKTID_IMR BIT(30) +#define B_BE_F2PCMD_RD_PKTID_IMR BIT(29) +#define B_BE_F2PCMD_ASSIGN_PKTID_IMR BIT(28) +#define B_BE_F2PCMD_USER_ALLC_IMR BIT(27) +#define B_BE_RX_SPF_U0_PKTID_IMR BIT(26) +#define B_BE_TX_SPF_U1_PKTID_IMR BIT(25) +#define B_BE_TX_SPF_U2_PKTID_IMR BIT(24) +#define B_BE_TX_SPF_U3_PKTID_IMR BIT(23) +#define B_BE_TX_RECORD_PKTID_IMR BIT(22) +#define B_BE_TWTSP_QSEL_IMR BIT(14) +#define B_BE_F2P_RLS_CTN_SEL_IMR BIT(13) +#define B_BE_BCNQ_ORDER_IMR BIT(12) +#define B_BE_Q_PKTID_IMR BIT(11) +#define B_BE_D_PKTID_IMR BIT(10) +#define B_BE_TXPRT_FULL_DROP_IMR BIT(9) +#define B_BE_F2PCMDRPT_FULL_DROP_IMR BIT(8) +#define B_BE_PTCL_IMR1_CLR (B_BE_F2PCMDRPT_FULL_DROP_IMR | \ + B_BE_TXPRT_FULL_DROP_IMR | \ + B_BE_D_PKTID_IMR | \ + B_BE_Q_PKTID_IMR | \ + B_BE_BCNQ_ORDER_IMR | \ + B_BE_F2P_RLS_CTN_SEL_IMR | \ + B_BE_TWTSP_QSEL_IMR | \ + B_BE_TX_RECORD_PKTID_IMR | \ + B_BE_TX_SPF_U3_PKTID_IMR | \ + B_BE_TX_SPF_U2_PKTID_IMR | \ + B_BE_TX_SPF_U1_PKTID_IMR | \ + B_BE_RX_SPF_U0_PKTID_IMR | \ + B_BE_F2PCMD_USER_ALLC_IMR | \ + B_BE_F2PCMD_ASSIGN_PKTID_IMR | \ + B_BE_F2PCMD_RD_PKTID_IMR | \ + B_BE_F2PCMD_PKTID_IMR) +#define B_BE_PTCL_IMR1_SET B_BE_F2PCMD_USER_ALLC_IMR + +#define R_BE_PTCL_ISR1 0x108CC +#define R_BE_PTCL_ISR1_C1 0x148CC +#define B_BE_F2PCMD_PKTID_ERR BIT(30) +#define B_BE_F2PCMD_RD_PKTID_ERR BIT(29) +#define B_BE_F2PCMD_ASSIGN_PKTID_ERR BIT(28) +#define B_BE_F2PCMD_USER_ALLC_ERR BIT(27) +#define B_BE_RX_SPF_U0_PKTID_ERR BIT(26) +#define B_BE_TX_SPF_U1_PKTID_ERR BIT(25) +#define B_BE_TX_SPF_U2_PKTID_ERR BIT(24) +#define B_BE_TX_SPF_U3_PKTID_ERR BIT(23) +#define B_BE_TX_RECORD_PKTID_ERR BIT(22) +#define B_BE_TWTSP_QSEL_ERR BIT(14) +#define B_BE_F2P_RLS_CTN_SEL_ERR BIT(13) +#define B_BE_BCNQ_ORDER_ERR BIT(12) +#define B_BE_Q_PKTID_ERR BIT(11) +#define B_BE_D_PKTID_ERR BIT(10) +#define B_BE_TXPRT_FULL_DROP_ERR BIT(9) +#define B_BE_F2PCMDRPT_FULL_DROP_ERR BIT(8) + +#define R_BE_RX_ERROR_FLAG 0x10C00 +#define R_BE_RX_ERROR_FLAG_C1 0x14C00 +#define B_BE_RX_CSI_NOT_RELEASE_ERROR BIT(31) +#define B_BE_RX_GET_NULL_PKT_ERROR BIT(30) +#define B_BE_RX_RU0_FSM_HANG_ERROR BIT(29) +#define B_BE_RX_RU1_FSM_HANG_ERROR BIT(28) +#define B_BE_RX_RU2_FSM_HANG_ERROR BIT(27) +#define B_BE_RX_RU3_FSM_HANG_ERROR BIT(26) +#define B_BE_RX_RU4_FSM_HANG_ERROR BIT(25) +#define B_BE_RX_RU5_FSM_HANG_ERROR BIT(24) +#define B_BE_RX_RU6_FSM_HANG_ERROR BIT(23) +#define B_BE_RX_RU7_FSM_HANG_ERROR BIT(22) +#define B_BE_RX_RXSTS_FSM_HANG_ERROR BIT(21) +#define B_BE_RX_CSI_FSM_HANG_ERROR BIT(20) +#define B_BE_RX_TXRPT_FSM_HANG_ERROR BIT(19) +#define B_BE_RX_F2PCMD_FSM_HANG_ERROR BIT(18) +#define B_BE_RX_RU0_ZERO_LENGTH_ERROR BIT(17) +#define B_BE_RX_RU1_ZERO_LENGTH_ERROR BIT(16) +#define B_BE_RX_RU2_ZERO_LENGTH_ERROR BIT(15) +#define B_BE_RX_RU3_ZERO_LENGTH_ERROR BIT(14) +#define B_BE_RX_RU4_ZERO_LENGTH_ERROR BIT(13) +#define B_BE_RX_RU5_ZERO_LENGTH_ERROR BIT(12) +#define B_BE_RX_RU6_ZERO_LENGTH_ERROR BIT(11) +#define B_BE_RX_RU7_ZERO_LENGTH_ERROR BIT(10) +#define B_BE_RX_RXSTS_ZERO_LENGTH_ERROR BIT(9) +#define B_BE_RX_CSI_ZERO_LENGTH_ERROR BIT(8) +#define B_BE_PLE_DATA_OPT_FSM_HANG BIT(7) +#define B_BE_PLE_RXDATA_REQUEST_BUFFER_FSM_HANG BIT(6) +#define B_BE_PLE_TXRPT_REQUEST_BUFFER_FSM_HANG BIT(5) +#define B_BE_PLE_WD_OPT_FSM_HANG BIT(4) +#define B_BE_PLE_ENQ_FSM_HANG BIT(3) +#define B_BE_RXDATA_ENQUE_ORDER_ERROR BIT(2) +#define B_BE_RXSTS_ENQUE_ORDER_ERROR BIT(1) +#define B_BE_RX_CSI_PKT_NUM_ERROR BIT(0) + +#define R_BE_RX_ERROR_FLAG_IMR 0x10C04 +#define R_BE_RX_ERROR_FLAG_IMR_C1 0x14C04 +#define B_BE_RX_CSI_NOT_RELEASE_ERROR_IMR BIT(31) +#define B_BE_RX_GET_NULL_PKT_ERROR_IMR BIT(30) +#define B_BE_RX_RU0_FSM_HANG_ERROR_IMR BIT(29) +#define B_BE_RX_RU1_FSM_HANG_ERROR_IMR BIT(28) +#define B_BE_RX_RU2_FSM_HANG_ERROR_IMR BIT(27) +#define B_BE_RX_RU3_FSM_HANG_ERROR_IMR BIT(26) +#define B_BE_RX_RU4_FSM_HANG_ERROR_IMR BIT(25) +#define B_BE_RX_RU5_FSM_HANG_ERROR_IMR BIT(24) +#define B_BE_RX_RU6_FSM_HANG_ERROR_IMR BIT(23) +#define B_BE_RX_RU7_FSM_HANG_ERROR_IMR BIT(22) +#define B_BE_RX_RXSTS_FSM_HANG_ERROR_IMR BIT(21) +#define B_BE_RX_CSI_FSM_HANG_ERROR_IMR BIT(20) +#define B_BE_RX_TXRPT_FSM_HANG_ERROR_IMR BIT(19) +#define B_BE_RX_F2PCMD_FSM_HANG_ERROR_IMR BIT(18) +#define B_BE_RX_RU0_ZERO_LENGTH_ERROR_IMR BIT(17) +#define B_BE_RX_RU1_ZERO_LENGTH_ERROR_IMR BIT(16) +#define B_BE_RX_RU2_ZERO_LENGTH_ERROR_IMR BIT(15) +#define B_BE_RX_RU3_ZERO_LENGTH_ERROR_IMR BIT(14) +#define B_BE_RX_RU4_ZERO_LENGTH_ERROR_IMR BIT(13) +#define B_BE_RX_RU5_ZERO_LENGTH_ERROR_IMR BIT(12) +#define B_BE_RX_RU6_ZERO_LENGTH_ERROR_IMR BIT(11) +#define B_BE_RX_RU7_ZERO_LENGTH_ERROR_IMR BIT(10) +#define B_BE_RX_RXSTS_ZERO_LENGTH_ERROR_IMR BIT(9) +#define B_BE_RX_CSI_ZERO_LENGTH_ERROR_IMR BIT(8) +#define B_BE_PLE_DATA_OPT_FSM_HANG_IMR BIT(7) +#define B_BE_PLE_RXDATA_REQUEST_BUFFER_FSM_HANG_IMR BIT(6) +#define B_BE_PLE_TXRPT_REQUEST_BUFFER_FSM_HANG_IMR BIT(5) +#define B_BE_PLE_WD_OPT_FSM_HANG_IMR BIT(4) +#define B_BE_PLE_ENQ_FSM_HANG_IMR BIT(3) +#define B_BE_RXDATA_ENQUE_ORDER_ERROR_IMR BIT(2) +#define B_BE_RXSTS_ENQUE_ORDER_ERROR_IMR BIT(1) +#define B_BE_RX_CSI_PKT_NUM_ERROR_IMR BIT(0) +#define B_BE_RX_ERROR_FLAG_IMR_CLR (B_BE_RX_RXSTS_ZERO_LENGTH_ERROR_IMR | \ + B_BE_RX_RU7_ZERO_LENGTH_ERROR_IMR | \ + B_BE_RX_RU6_ZERO_LENGTH_ERROR_IMR | \ + B_BE_RX_RU5_ZERO_LENGTH_ERROR_IMR | \ + B_BE_RX_RU4_ZERO_LENGTH_ERROR_IMR | \ + B_BE_RX_RU3_ZERO_LENGTH_ERROR_IMR | \ + B_BE_RX_RU2_ZERO_LENGTH_ERROR_IMR | \ + B_BE_RX_RU1_ZERO_LENGTH_ERROR_IMR | \ + B_BE_RX_RU0_ZERO_LENGTH_ERROR_IMR | \ + B_BE_RX_F2PCMD_FSM_HANG_ERROR_IMR | \ + B_BE_RX_TXRPT_FSM_HANG_ERROR_IMR | \ + B_BE_RX_CSI_FSM_HANG_ERROR_IMR | \ + B_BE_RX_RXSTS_FSM_HANG_ERROR_IMR | \ + B_BE_RX_RU7_FSM_HANG_ERROR_IMR | \ + B_BE_RX_RU6_FSM_HANG_ERROR_IMR | \ + B_BE_RX_RU5_FSM_HANG_ERROR_IMR | \ + B_BE_RX_RU4_FSM_HANG_ERROR_IMR | \ + B_BE_RX_RU3_FSM_HANG_ERROR_IMR | \ + B_BE_RX_RU2_FSM_HANG_ERROR_IMR | \ + B_BE_RX_RU1_FSM_HANG_ERROR_IMR | \ + B_BE_RX_RU0_FSM_HANG_ERROR_IMR | \ + B_BE_RX_GET_NULL_PKT_ERROR_IMR) +#define B_BE_RX_ERROR_FLAG_IMR_SET (B_BE_RX_RXSTS_ZERO_LENGTH_ERROR_IMR | \ + B_BE_RX_RU7_ZERO_LENGTH_ERROR_IMR | \ + B_BE_RX_RU6_ZERO_LENGTH_ERROR_IMR | \ + B_BE_RX_RU5_ZERO_LENGTH_ERROR_IMR | \ + B_BE_RX_RU4_ZERO_LENGTH_ERROR_IMR | \ + B_BE_RX_RU3_ZERO_LENGTH_ERROR_IMR | \ + B_BE_RX_RU2_ZERO_LENGTH_ERROR_IMR | \ + B_BE_RX_RU1_ZERO_LENGTH_ERROR_IMR | \ + B_BE_RX_RU0_ZERO_LENGTH_ERROR_IMR | \ + B_BE_RX_F2PCMD_FSM_HANG_ERROR_IMR | \ + B_BE_RX_TXRPT_FSM_HANG_ERROR_IMR | \ + B_BE_RX_CSI_FSM_HANG_ERROR_IMR | \ + B_BE_RX_RXSTS_FSM_HANG_ERROR_IMR | \ + B_BE_RX_RU7_FSM_HANG_ERROR_IMR | \ + B_BE_RX_RU6_FSM_HANG_ERROR_IMR | \ + B_BE_RX_RU5_FSM_HANG_ERROR_IMR | \ + B_BE_RX_RU4_FSM_HANG_ERROR_IMR | \ + B_BE_RX_RU3_FSM_HANG_ERROR_IMR | \ + B_BE_RX_RU2_FSM_HANG_ERROR_IMR | \ + B_BE_RX_RU1_FSM_HANG_ERROR_IMR | \ + B_BE_RX_RU0_FSM_HANG_ERROR_IMR | \ + B_BE_RX_GET_NULL_PKT_ERROR_IMR) + +#define R_BE_TX_ERROR_FLAG 0x10C6C +#define R_BE_TX_ERROR_FLAG_C1 0x14C6C +#define B_BE_TX_RU0_FSM_HANG_ERROR BIT(31) +#define B_BE_TX_RU1_FSM_HANG_ERROR BIT(30) +#define B_BE_TX_RU2_FSM_HANG_ERROR BIT(29) +#define B_BE_TX_RU3_FSM_HANG_ERROR BIT(28) +#define B_BE_TX_RU4_FSM_HANG_ERROR BIT(27) +#define B_BE_TX_RU5_FSM_HANG_ERROR BIT(26) +#define B_BE_TX_RU6_FSM_HANG_ERROR BIT(25) +#define B_BE_TX_RU7_FSM_HANG_ERROR BIT(24) +#define B_BE_TX_RU8_FSM_HANG_ERROR BIT(23) +#define B_BE_TX_RU9_FSM_HANG_ERROR BIT(22) +#define B_BE_TX_RU10_FSM_HANG_ERROR BIT(21) +#define B_BE_TX_RU11_FSM_HANG_ERROR BIT(20) +#define B_BE_TX_RU12_FSM_HANG_ERROR BIT(19) +#define B_BE_TX_RU13_FSM_HANG_ERROR BIT(18) +#define B_BE_TX_RU14_FSM_HANG_ERROR BIT(17) +#define B_BE_TX_RU15_FSM_HANG_ERROR BIT(16) +#define B_BE_TX_CSI_FSM_HANG_ERROR BIT(15) +#define B_BE_TX_WD_PLD_ID_FSM_HANG_ERROR BIT(14) + +#define R_BE_TX_ERROR_FLAG_IMR 0x10C70 +#define R_BE_TX_ERROR_FLAG_IMR_C1 0x14C70 +#define B_BE_TX_RU0_FSM_HANG_ERROR_IMR BIT(31) +#define B_BE_TX_RU1_FSM_HANG_ERROR_IMR BIT(30) +#define B_BE_TX_RU2_FSM_HANG_ERROR_IMR BIT(29) +#define B_BE_TX_RU3_FSM_HANG_ERROR_IMR BIT(28) +#define B_BE_TX_RU4_FSM_HANG_ERROR_IMR BIT(27) +#define B_BE_TX_RU5_FSM_HANG_ERROR_IMR BIT(26) +#define B_BE_TX_RU6_FSM_HANG_ERROR_IMR BIT(25) +#define B_BE_TX_RU7_FSM_HANG_ERROR_IMR BIT(24) +#define B_BE_TX_RU8_FSM_HANG_ERROR_IMR BIT(23) +#define B_BE_TX_RU9_FSM_HANG_ERROR_IMR BIT(22) +#define B_BE_TX_RU10_FSM_HANG_ERROR_IMR BIT(21) +#define B_BE_TX_RU11_FSM_HANG_ERROR_IMR BIT(20) +#define B_BE_TX_RU12_FSM_HANG_ERROR_IMR BIT(19) +#define B_BE_TX_RU13_FSM_HANG_ERROR_IMR BIT(18) +#define B_BE_TX_RU14_FSM_HANG_ERROR_IMR BIT(17) +#define B_BE_TX_RU15_FSM_HANG_ERROR_IMR BIT(16) +#define B_BE_TX_CSI_FSM_HANG_ERROR_IMR BIT(15) +#define B_BE_TX_WD_PLD_ID_FSM_HANG_ERROR_IMR BIT(14) +#define B_BE_TX_ERROR_FLAG_IMR_CLR (B_BE_TX_WD_PLD_ID_FSM_HANG_ERROR_IMR | \ + B_BE_TX_CSI_FSM_HANG_ERROR_IMR | \ + B_BE_TX_RU15_FSM_HANG_ERROR_IMR | \ + B_BE_TX_RU14_FSM_HANG_ERROR_IMR | \ + B_BE_TX_RU13_FSM_HANG_ERROR_IMR | \ + B_BE_TX_RU12_FSM_HANG_ERROR_IMR | \ + B_BE_TX_RU11_FSM_HANG_ERROR_IMR | \ + B_BE_TX_RU10_FSM_HANG_ERROR_IMR | \ + B_BE_TX_RU9_FSM_HANG_ERROR_IMR | \ + B_BE_TX_RU8_FSM_HANG_ERROR_IMR | \ + B_BE_TX_RU7_FSM_HANG_ERROR_IMR | \ + B_BE_TX_RU6_FSM_HANG_ERROR_IMR | \ + B_BE_TX_RU5_FSM_HANG_ERROR_IMR | \ + B_BE_TX_RU4_FSM_HANG_ERROR_IMR | \ + B_BE_TX_RU3_FSM_HANG_ERROR_IMR | \ + B_BE_TX_RU2_FSM_HANG_ERROR_IMR | \ + B_BE_TX_RU1_FSM_HANG_ERROR_IMR | \ + B_BE_TX_RU0_FSM_HANG_ERROR_IMR) +#define B_BE_TX_ERROR_FLAG_IMR_SET (B_BE_TX_WD_PLD_ID_FSM_HANG_ERROR_IMR | \ + B_BE_TX_CSI_FSM_HANG_ERROR_IMR | \ + B_BE_TX_RU15_FSM_HANG_ERROR_IMR | \ + B_BE_TX_RU14_FSM_HANG_ERROR_IMR | \ + B_BE_TX_RU13_FSM_HANG_ERROR_IMR | \ + B_BE_TX_RU12_FSM_HANG_ERROR_IMR | \ + B_BE_TX_RU11_FSM_HANG_ERROR_IMR | \ + B_BE_TX_RU10_FSM_HANG_ERROR_IMR | \ + B_BE_TX_RU9_FSM_HANG_ERROR_IMR | \ + B_BE_TX_RU8_FSM_HANG_ERROR_IMR | \ + B_BE_TX_RU7_FSM_HANG_ERROR_IMR | \ + B_BE_TX_RU6_FSM_HANG_ERROR_IMR | \ + B_BE_TX_RU5_FSM_HANG_ERROR_IMR | \ + B_BE_TX_RU4_FSM_HANG_ERROR_IMR | \ + B_BE_TX_RU3_FSM_HANG_ERROR_IMR | \ + B_BE_TX_RU2_FSM_HANG_ERROR_IMR | \ + B_BE_TX_RU1_FSM_HANG_ERROR_IMR | \ + B_BE_TX_RU0_FSM_HANG_ERROR_IMR) + +#define R_BE_RX_ERROR_FLAG_1 0x10C84 +#define R_BE_RX_ERROR_FLAG_1_C1 0x14C84 +#define B_BE_RX_RU8_FSM_HANG_ERROR BIT(29) +#define B_BE_RX_RU9_FSM_HANG_ERROR BIT(28) +#define B_BE_RX_RU10_FSM_HANG_ERROR BIT(27) +#define B_BE_RX_RU11_FSM_HANG_ERROR BIT(26) +#define B_BE_RX_RU12_FSM_HANG_ERROR BIT(25) +#define B_BE_RX_RU13_FSM_HANG_ERROR BIT(24) +#define B_BE_RX_RU14_FSM_HANG_ERROR BIT(23) +#define B_BE_RX_RU15_FSM_HANG_ERROR BIT(22) +#define B_BE_RX_RU8_ZERO_LENGTH_ERROR BIT(17) +#define B_BE_RX_RU9_ZERO_LENGTH_ERROR BIT(16) +#define B_BE_RX_RU10_ZERO_LENGTH_ERROR BIT(15) +#define B_BE_RX_RU11_ZERO_LENGTH_ERROR BIT(14) +#define B_BE_RX_RU12_ZERO_LENGTH_ERROR BIT(13) +#define B_BE_RX_RU13_ZERO_LENGTH_ERROR BIT(12) +#define B_BE_RX_RU14_ZERO_LENGTH_ERROR BIT(11) +#define B_BE_RX_RU15_ZERO_LENGTH_ERROR BIT(10) + +#define R_BE_RX_ERROR_FLAG_IMR_1 0x10C88 +#define R_BE_RX_ERROR_FLAG_IMR_1_C1 0x14C88 +#define B_BE_RX_RU8_FSM_HANG_ERROR_IMR BIT(29) +#define B_BE_RX_RU9_FSM_HANG_ERROR_IMR BIT(28) +#define B_BE_RX_RU10_FSM_HANG_ERROR_IMR BIT(27) +#define B_BE_RX_RU11_FSM_HANG_ERROR_IMR BIT(26) +#define B_BE_RX_RU12_FSM_HANG_ERROR_IMR BIT(25) +#define B_BE_RX_RU13_FSM_HANG_ERROR_IMR BIT(24) +#define B_BE_RX_RU14_FSM_HANG_ERROR_IMR BIT(23) +#define B_BE_RX_RU15_FSM_HANG_ERROR_IMR BIT(22) +#define B_BE_RX_RU8_ZERO_LENGTH_ERROR_IMR BIT(17) +#define B_BE_RX_RU9_ZERO_LENGTH_ERROR_IMR BIT(16) +#define B_BE_RX_RU10_ZERO_LENGTH_ERROR_IMR BIT(15) +#define B_BE_RX_RU11_ZERO_LENGTH_ERROR_IMR BIT(14) +#define B_BE_RX_RU12_ZERO_LENGTH_ERROR_IMR BIT(13) +#define B_BE_RX_RU13_ZERO_LENGTH_ERROR_IMR BIT(12) +#define B_BE_RX_RU14_ZERO_LENGTH_ERROR_IMR BIT(11) +#define B_BE_RX_RU15_ZERO_LENGTH_ERROR_IMR BIT(10) +#define B_BE_TX_ERROR_FLAG_IMR_1_CLR (B_BE_RX_RU8_FSM_HANG_ERROR_IMR | \ + B_BE_RX_RU9_FSM_HANG_ERROR_IMR | \ + B_BE_RX_RU10_FSM_HANG_ERROR_IMR | \ + B_BE_RX_RU11_FSM_HANG_ERROR_IMR | \ + B_BE_RX_RU12_FSM_HANG_ERROR_IMR | \ + B_BE_RX_RU13_FSM_HANG_ERROR_IMR | \ + B_BE_RX_RU14_FSM_HANG_ERROR_IMR | \ + B_BE_RX_RU15_FSM_HANG_ERROR_IMR | \ + B_BE_RX_RU8_ZERO_LENGTH_ERROR_IMR | \ + B_BE_RX_RU9_ZERO_LENGTH_ERROR_IMR | \ + B_BE_RX_RU10_ZERO_LENGTH_ERROR_IMR | \ + B_BE_RX_RU11_ZERO_LENGTH_ERROR_IMR | \ + B_BE_RX_RU12_ZERO_LENGTH_ERROR_IMR | \ + B_BE_RX_RU13_ZERO_LENGTH_ERROR_IMR | \ + B_BE_RX_RU14_ZERO_LENGTH_ERROR_IMR | \ + B_BE_RX_RU15_ZERO_LENGTH_ERROR_IMR) +#define B_BE_TX_ERROR_FLAG_IMR_1_SET (B_BE_RX_RU8_FSM_HANG_ERROR_IMR | \ + B_BE_RX_RU9_FSM_HANG_ERROR_IMR | \ + B_BE_RX_RU10_FSM_HANG_ERROR_IMR | \ + B_BE_RX_RU11_FSM_HANG_ERROR_IMR | \ + B_BE_RX_RU12_FSM_HANG_ERROR_IMR | \ + B_BE_RX_RU13_FSM_HANG_ERROR_IMR | \ + B_BE_RX_RU14_FSM_HANG_ERROR_IMR | \ + B_BE_RX_RU15_FSM_HANG_ERROR_IMR | \ + B_BE_RX_RU8_ZERO_LENGTH_ERROR_IMR | \ + B_BE_RX_RU9_ZERO_LENGTH_ERROR_IMR | \ + B_BE_RX_RU10_ZERO_LENGTH_ERROR_IMR | \ + B_BE_RX_RU11_ZERO_LENGTH_ERROR_IMR | \ + B_BE_RX_RU12_ZERO_LENGTH_ERROR_IMR | \ + B_BE_RX_RU13_ZERO_LENGTH_ERROR_IMR | \ + B_BE_RX_RU14_ZERO_LENGTH_ERROR_IMR | \ + B_BE_RX_RU15_ZERO_LENGTH_ERROR_IMR) + #define R_BE_WMTX_MOREDATA_TSFT_STMP_CTL 0x10E08 #define R_BE_WMTX_MOREDATA_TSFT_STMP_CTL_C1 0x14E08 #define B_BE_TSFT_OFS_MASK GENMASK(31, 16) @@ -4371,6 +6353,124 @@ #define B_BE_UPD_HGQMD BIT(1) #define B_BE_UPD_TIMIE BIT(0) +#define R_BE_RSP_CHK_SIG 0x11000 +#define R_BE_RSP_CHK_SIG_C1 0x15000 +#define B_BE_RSP_STATIC_RTS_CHK_SERV_BW_EN BIT(30) +#define B_BE_RSP_TBPPDU_CHK_PWR BIT(29) +#define B_BE_RESP_PAIR_MACID_LEN_EN BIT(25) +#define B_BE_RESP_TX_ABORT_TEST_EN BIT(24) +#define B_BE_RESP_ER_SU_RU106_EN BIT(23) +#define B_BE_RESP_ER_SU_EN BIT(22) +#define B_BE_TXDATA_END_PS_OPT BIT(18) +#define B_BE_CHECK_SOUNDING_SEQ BIT(17) +#define B_BE_RXBA_IGNOREA2 BIT(16) +#define B_BE_ACKTO_CCK_MASK GENMASK(15, 8) +#define B_BE_ACKTO_MASK GENMASK(8, 0) + +#define R_BE_TRXPTCL_RESP_0 0x11004 +#define R_BE_TRXPTCL_RESP_0_C1 0x15004 +#define B_BE_WMAC_RESP_STBC_EN BIT(31) +#define B_BE_WMAC_RXFTM_TXACK_SB BIT(30) +#define B_BE_WMAC_RXFTM_TXACKBWEQ BIT(29) +#define B_BE_RESP_TB_CHK_TXTIME BIT(24) +#define B_BE_RSP_CHK_CCA BIT(23) +#define B_BE_WMAC_LDPC_EN BIT(22) +#define B_BE_WMAC_SGIEN BIT(21) +#define B_BE_WMAC_SPLCPEN BIT(20) +#define B_BE_RESP_EHT_MCS15_REF BIT(19) +#define B_BE_RESP_EHT_MCS14_REF BIT(18) +#define B_BE_WMAC_BESP_EARLY_TXBA BIT(17) +#define B_BE_WMAC_MBA_DUR_FORCE BIT(16) +#define B_BE_WMAC_SPEC_SIFS_OFDM_MASK GENMASK(15, 8) +#define WMAC_SPEC_SIFS_OFDM_1115E 0x11 +#define B_BE_WMAC_SPEC_SIFS_CCK_MASK GENMASK(7, 0) + +#define R_BE_TRXPTCL_ERROR_INDICA_MASK 0x110BC +#define R_BE_TRXPTCL_ERROR_INDICA_MASK_C1 0x150BC +#define B_BE_WMAC_FTM_TIMEOUT_MODE BIT(30) +#define B_BE_WMAC_FTM_TIMEOUT_THR_MASK GENMASK(29, 24) +#define B_BE_WMAC_MODE BIT(22) +#define B_BE_WMAC_TIMETOUT_THR_MASK GENMASK(21, 16) +#define B_BE_RMAC_BFMER BIT(9) +#define B_BE_RMAC_FTM BIT(8) +#define B_BE_RMAC_CSI BIT(7) +#define B_BE_TMAC_MIMO_CTRL BIT(6) +#define B_BE_TMAC_RXTB BIT(5) +#define B_BE_TMAC_HWSIGB_GEN BIT(4) +#define B_BE_TMAC_TXPLCP BIT(3) +#define B_BE_TMAC_RESP BIT(2) +#define B_BE_TMAC_TXCTL BIT(1) +#define B_BE_TMAC_MACTX BIT(0) +#define B_BE_TRXPTCL_ERROR_INDICA_MASK_CLR (B_BE_TMAC_MACTX | \ + B_BE_TMAC_TXCTL | \ + B_BE_TMAC_RESP | \ + B_BE_TMAC_TXPLCP | \ + B_BE_TMAC_HWSIGB_GEN | \ + B_BE_TMAC_RXTB | \ + B_BE_TMAC_MIMO_CTRL | \ + B_BE_RMAC_CSI | \ + B_BE_RMAC_FTM | \ + B_BE_RMAC_BFMER) +#define B_BE_TRXPTCL_ERROR_INDICA_MASK_SET (B_BE_TMAC_MACTX | \ + B_BE_TMAC_TXCTL | \ + B_BE_TMAC_RESP | \ + B_BE_TMAC_TXPLCP | \ + B_BE_TMAC_HWSIGB_GEN | \ + B_BE_TMAC_RXTB | \ + B_BE_TMAC_MIMO_CTRL | \ + B_BE_RMAC_CSI | \ + B_BE_RMAC_FTM | \ + B_BE_RMAC_BFMER) + +#define R_BE_TRXPTCL_ERROR_INDICA 0x110C0 +#define R_BE_TRXPTCL_ERROR_INDICA_C1 0x150C0 +#define B_BE_BFMER_ERR_FLAG BIT(9) +#define B_BE_FTM_ERROR_FLAG_CLR BIT(8) +#define B_BE_CSI_ERROR_FLAG_CLR BIT(7) +#define B_BE_MIMOCTRL_ERROR_FLAG_CLR BIT(6) +#define B_BE_RXTB_ERROR_FLAG_CLR BIT(5) +#define B_BE_HWSIGB_GEN_ERROR_FLAG_CLR BIT(4) +#define B_BE_TXPLCP_ERROR_FLAG_CLR BIT(3) +#define B_BE_RESP_ERROR_FLAG_CLR BIT(2) +#define B_BE_TXCTL_ERROR_FLAG_CLR BIT(1) +#define B_BE_MACTX_ERROR_FLAG_CLR BIT(0) + +#define R_BE_DBGSEL_TRXPTCL 0x110F4 +#define R_BE_DBGSEL_TRXPTCL_C1 0x150F4 +#define B_BE_WMAC_CHNSTS_STATE_MASK GENMASK(19, 16) +#define B_BE_DBGSEL_TRIGCMD_SEL_MASK GENMASK(11, 8) +#define B_BE_DBGSEL_TRXPTCL_MASK GENMASK(7, 0) + +#define R_BE_PHYINFO_ERR_IMR_V1 0x110F8 +#define R_BE_PHYINFO_ERR_IMR_V1_C1 0x150F8 +#define B_BE_PHYINTF_RXTB_WIDTH_MASK GENMASK(31, 30) +#define B_BE_PHYINTF_RXTB_EN_PHASE_MASK GENMASK(29, 28) +#define B_BE_PHYINTF_MIMO_WIDTH_MASK GENMASK(27, 26) +#define B_BE_PHYINTF_MIMO_EN_PHASE_MASK GENMASK(25, 24) +#define B_BE_PHYINTF_TIMEOUT_THR_V1_MASK GENMASK(21, 16) +#define B_BE_CSI_ON_TIMEOUT_EN BIT(5) +#define B_BE_STS_ON_TIMEOUT_EN BIT(4) +#define B_BE_DATA_ON_TIMEOUT_EN BIT(3) +#define B_BE_OFDM_CCA_TIMEOUT_EN BIT(2) +#define B_BE_CCK_CCA_TIMEOUT_EN BIT(1) +#define B_BE_PHY_TXON_TIMEOUT_EN BIT(0) +#define B_BE_PHYINFO_ERR_IMR_V1_CLR (B_BE_PHY_TXON_TIMEOUT_EN | \ + B_BE_CCK_CCA_TIMEOUT_EN | \ + B_BE_OFDM_CCA_TIMEOUT_EN | \ + B_BE_DATA_ON_TIMEOUT_EN | \ + B_BE_STS_ON_TIMEOUT_EN | \ + B_BE_CSI_ON_TIMEOUT_EN) +#define B_BE_PHYINFO_ERR_IMR_V1_SET 0 + +#define R_BE_PHYINFO_ERR_ISR 0x110FC +#define R_BE_PHYINFO_ERR_ISR_C1 0x150FC +#define B_BE_CSI_ON_TIMEOUT_ERR BIT(5) +#define B_BE_STS_ON_TIMEOUT_ERR BIT(4) +#define B_BE_DATA_ON_TIMEOUT_ERR BIT(3) +#define B_BE_OFDM_CCA_TIMEOUT_ERR BIT(2) +#define B_BE_CCK_CCA_TIMEOUT_ERR BIT(1) +#define B_BE_PHY_TXON_TIMEOUT_ERR BIT(0) + #define R_BE_BFMEE_RESP_OPTION 0x11180 #define R_BE_BFMEE_RESP_OPTION_C1 0x15180 #define B_BE_BFMEE_CSI_SEC_TYPE_SH 20 @@ -4451,6 +6551,90 @@ #define B_BE_CSIPRT_HESU_AID_EN BIT(25) #define B_BE_CSIPRT_VHTSU_AID_EN BIT(24) +#define R_BE_RX_ERR_ISR 0x114F4 +#define R_BE_RX_ERR_ISR_C1 0x154F4 +#define B_BE_RX_ERR_TRIG_ACT_TO BIT(9) +#define B_BE_RX_ERR_STS_ACT_TO BIT(8) +#define B_BE_RX_ERR_CSI_ACT_TO BIT(7) +#define B_BE_RX_ERR_ACT_TO BIT(6) +#define B_BE_CSI_DATAON_ASSERT_TO BIT(5) +#define B_BE_DATAON_ASSERT_TO BIT(4) +#define B_BE_CCA_ASSERT_TO BIT(3) +#define B_BE_RX_ERR_DMA_TO BIT(2) +#define B_BE_RX_ERR_DATA_TO BIT(1) +#define B_BE_RX_ERR_CCA_TO BIT(0) + +#define R_BE_RX_ERR_IMR 0x114F8 +#define R_BE_RX_ERR_IMR_C1 0x154F8 +#define B_BE_RX_ERR_TRIG_ACT_TO_MSK BIT(9) +#define B_BE_RX_ERR_STS_ACT_TO_MSK BIT(8) +#define B_BE_RX_ERR_CSI_ACT_TO_MSK BIT(7) +#define B_BE_RX_ERR_ACT_TO_MSK BIT(6) +#define B_BE_CSI_DATAON_ASSERT_TO_MSK BIT(5) +#define B_BE_DATAON_ASSERT_TO_MSK BIT(4) +#define B_BE_CCA_ASSERT_TO_MSK BIT(3) +#define B_BE_RX_ERR_DMA_TO_MSK BIT(2) +#define B_BE_RX_ERR_DATA_TO_MSK BIT(1) +#define B_BE_RX_ERR_CCA_TO_MSK BIT(0) +#define B_BE_RX_ERR_IMR_CLR (B_BE_RX_ERR_CCA_TO_MSK | \ + B_BE_RX_ERR_DATA_TO_MSK | \ + B_BE_RX_ERR_DMA_TO_MSK | \ + B_BE_CCA_ASSERT_TO_MSK | \ + B_BE_DATAON_ASSERT_TO_MSK | \ + B_BE_CSI_DATAON_ASSERT_TO_MSK | \ + B_BE_RX_ERR_ACT_TO_MSK | \ + B_BE_RX_ERR_CSI_ACT_TO_MSK | \ + B_BE_RX_ERR_STS_ACT_TO_MSK | \ + B_BE_RX_ERR_TRIG_ACT_TO_MSK) +#define B_BE_RX_ERR_IMR_SET (B_BE_RX_ERR_ACT_TO_MSK | \ + B_BE_RX_ERR_STS_ACT_TO_MSK | \ + B_BE_RX_ERR_TRIG_ACT_TO_MSK) + +#define R_BE_RESP_IMR 0x11884 +#define R_BE_RESP_IMR_C1 0x15884 +#define B_BE_RESP_TBL_FLAG_ERR_ISR_EN BIT(17) +#define B_BE_RESP_SEC_DOUBLE_HIT_ERR_ISR_EN BIT(16) +#define B_BE_RESP_WRPTR_CROSS_ERR_ISR_EN BIT(15) +#define B_BE_RESP_TOO_MANY_PLD_ERR_ISR_EN BIT(14) +#define B_BE_RESP_TXDMA_READ_DATA_ERR_ISR_EN BIT(13) +#define B_BE_RESP_PLDID_RDY_ERR_ISR_EN BIT(12) +#define B_BE_RESP_RX_OVERWRITE_ERR_ISR_EN BIT(11) +#define B_BE_RESP_RXDMA_WRPTR_INVLD_ERR_ISR_EN BIT(10) +#define B_BE_RESP_RXDMA_REQ_INVLD_ERR_ISR_EN BIT(9) +#define B_BE_RESP_RXDMA_REQ_MACID_ERR_ISR_EN BIT(8) +#define B_BE_RESP_TXCMD_TX_ST_ABORT_ERR_ISR_EN BIT(6) +#define B_BE_RESP_TXCMD_DMAC_PROC_ERR_ISR_EN BIT(5) +#define B_BE_RESP_TXCMD_TBL_ERR_ISR_EN BIT(4) +#define B_BE_RESP_INITCMD_RX_ST_ABORT_ERR_ISR_EN BIT(3) +#define B_BE_RESP_INITCMD_RESERVD_PAGE_ABORT_ERR_ISR_EN BIT(2) +#define B_BE_RESP_INITCMD_TX_ST_ABORT_ERR_ISR_EN BIT(1) +#define B_BE_RESP_DMAC_PROC_ERR_ISR_EN BIT(0) +#define B_BE_RESP_IMR_CLR (B_BE_RESP_DMAC_PROC_ERR_ISR_EN | \ + B_BE_RESP_INITCMD_TX_ST_ABORT_ERR_ISR_EN | \ + B_BE_RESP_INITCMD_RX_ST_ABORT_ERR_ISR_EN | \ + B_BE_RESP_TXCMD_TBL_ERR_ISR_EN | \ + B_BE_RESP_TXCMD_DMAC_PROC_ERR_ISR_EN | \ + B_BE_RESP_TXCMD_TX_ST_ABORT_ERR_ISR_EN | \ + B_BE_RESP_RXDMA_REQ_MACID_ERR_ISR_EN | \ + B_BE_RESP_RXDMA_REQ_INVLD_ERR_ISR_EN | \ + B_BE_RESP_RXDMA_WRPTR_INVLD_ERR_ISR_EN | \ + B_BE_RESP_RX_OVERWRITE_ERR_ISR_EN | \ + B_BE_RESP_PLDID_RDY_ERR_ISR_EN | \ + B_BE_RESP_TXDMA_READ_DATA_ERR_ISR_EN | \ + B_BE_RESP_TOO_MANY_PLD_ERR_ISR_EN | \ + B_BE_RESP_WRPTR_CROSS_ERR_ISR_EN | \ + B_BE_RESP_SEC_DOUBLE_HIT_ERR_ISR_EN) +#define B_BE_RESP_IMR_SET (B_BE_RESP_DMAC_PROC_ERR_ISR_EN | \ + B_BE_RESP_INITCMD_TX_ST_ABORT_ERR_ISR_EN | \ + B_BE_RESP_INITCMD_RX_ST_ABORT_ERR_ISR_EN | \ + B_BE_RESP_TXCMD_TBL_ERR_ISR_EN | \ + B_BE_RESP_TXCMD_DMAC_PROC_ERR_ISR_EN | \ + B_BE_RESP_TXCMD_TX_ST_ABORT_ERR_ISR_EN | \ + B_BE_RESP_RX_OVERWRITE_ERR_ISR_EN | \ + B_BE_RESP_PLDID_RDY_ERR_ISR_EN | \ + B_BE_RESP_WRPTR_CROSS_ERR_ISR_EN | \ + B_BE_RESP_SEC_DOUBLE_HIT_ERR_ISR_EN) + #define R_BE_PWR_MODULE 0x11900 #define R_BE_PWR_MODULE_C1 0x15900 @@ -4462,6 +6646,17 @@ #define R_BE_PWR_RU_LMT 0x12048 #define R_BE_PWR_RU_LMT_MAX 0x120E4 +#define R_BE_C0_TXPWR_IMR 0x128E0 +#define R_BE_C0_TXPWR_IMR_C1 0x168E0 +#define B_BE_FSM_TIMEOUT_ERR_INT_EN BIT(0) +#define B_BE_C0_TXPWR_IMR_CLR B_BE_FSM_TIMEOUT_ERR_INT_EN +#define B_BE_C0_TXPWR_IMR_SET B_BE_FSM_TIMEOUT_ERR_INT_EN + +#define R_BE_TXPWR_ERR_FLAG 0x128E4 +#define R_BE_TXPWR_ERR_IMR 0x128E0 +#define R_BE_TXPWR_ERR_FLAG_C1 0x158E4 +#define R_BE_TXPWR_ERR_IMR_C1 0x158E0 + #define CMAC1_START_ADDR_BE 0x14000 #define CMAC1_END_ADDR_BE 0x17FFF @@ -4727,6 +6922,11 @@ #define B_P0_RSTB_WATCH_DOG BIT(0) #define B_P1_RSTB_WATCH_DOG BIT(1) #define B_UPD_P0_EN BIT(31) +#define R_SPOOF_CG 0x00B4 +#define B_SPOOF_CG_EN BIT(17) +#define R_DFS_FFT_CG 0x00B8 +#define B_DFS_CG_EN BIT(1) +#define B_DFS_FFT_EN BIT(0) #define R_ANAPAR_PW15 0x030C #define B_ANAPAR_PW15 GENMASK(31, 24) #define B_ANAPAR_PW15_H GENMASK(27, 24) @@ -4789,6 +6989,8 @@ #define R_PHY_STS_BITMAP_HT 0x076C #define R_PHY_STS_BITMAP_VHT 0x0770 #define R_PHY_STS_BITMAP_HE 0x0774 +#define R_EDCCA_RPTREG_SEL_BE 0x078C +#define B_EDCCA_RPTREG_SEL_BE_MSK GENMASK(22, 20) #define R_PMAC_GNT 0x0980 #define B_PMAC_GNT_TXEN BIT(0) #define B_PMAC_GNT_RXEN BIT(16) @@ -4848,12 +7050,18 @@ #define B_IOQ_IQK_DPK_EN BIT(1) #define R_GNT_BT_WGT_EN 0x0C6C #define B_GNT_BT_WGT_EN BIT(21) +#define R_TX_COLLISION_T2R_ST 0x0C70 +#define B_TX_COLLISION_T2R_ST_M GENMASK(25, 20) +#define R_TXGATING 0x0C74 +#define B_TXGATING_EN BIT(4) #define R_PD_ARBITER_OFF 0x0C80 #define B_PD_ARBITER_OFF BIT(31) #define R_SNDCCA_A1 0x0C9C #define B_SNDCCA_A1_EN GENMASK(19, 12) #define R_SNDCCA_A2 0x0CA0 #define B_SNDCCA_A2_VAL GENMASK(19, 12) +#define R_TX_COLLISION_T2R_ST_BE 0x0CC8 +#define B_TX_COLLISION_T2R_ST_BE_M GENMASK(13, 8) #define R_RXHT_MCS_LIMIT 0x0D18 #define B_RXHT_MCS_LIMIT GENMASK(9, 8) #define R_RXVHT_MCS_LIMIT 0x0D18 @@ -4872,6 +7080,10 @@ #define R_BRK_ASYNC_RST_EN_1 0x0DC0 #define R_BRK_ASYNC_RST_EN_2 0x0DC4 #define R_BRK_ASYNC_RST_EN_3 0x0DC8 +#define R_CTLTOP 0x1008 +#define B_CTLTOP_ON BIT(23) +#define B_CTLTOP_VAL GENMASK(15, 12) +#define R_EDCCA_RPT_SEL_BE 0x10CC #define R_S0_HW_SI_DIS 0x1200 #define B_S0_HW_SI_DIS_W_R_TRIG GENMASK(30, 28) #define R_P0_RXCK 0x12A0 @@ -4903,6 +7115,14 @@ #define R_CFO_COMP_SEG0_H 0x1388 #define R_CFO_COMP_SEG0_CTRL 0x138C #define R_DBG32_D 0x1730 +#define R_EDCCA_RPT_A 0x1738 +#define R_EDCCA_RPT_B 0x173c +#define B_EDCCA_RPT_B_FB BIT(7) +#define B_EDCCA_RPT_B_P20 BIT(6) +#define B_EDCCA_RPT_B_S20 BIT(5) +#define B_EDCCA_RPT_B_S40 BIT(4) +#define B_EDCCA_RPT_B_S80 BIT(3) +#define B_EDCCA_RPT_B_PATH_MASK GENMASK(2, 1) #define R_SWSI_V1 0x174C #define B_SWSI_W_BUSY_V1 BIT(24) #define B_SWSI_R_BUSY_V1 BIT(25) @@ -4964,6 +7184,8 @@ #define R_S0_ADDCK 0x1E00 #define B_S0_ADDCK_I GENMASK(9, 0) #define B_S0_ADDCK_Q GENMASK(19, 10) +#define R_EDCCA_RPT_SEL 0x20CC +#define B_EDCCA_RPT_SEL_MSK GENMASK(2, 0) #define R_ADC_FIFO 0x20fc #define B_ADC_FIFO_RST GENMASK(31, 24) #define B_ADC_FIFO_RXK GENMASK(31, 16) @@ -5010,6 +7232,8 @@ #define B_DBCC_80P80_SEL_EVM_RPT2_EN BIT(0) #define R_P1_EN_SOUND_WO_NDP 0x2D7C #define B_P1_EN_SOUND_WO_NDP BIT(1) +#define R_EDCCA_RPT_A_BE 0x2E38 +#define R_EDCCA_RPT_B_BE 0x2E3C #define R_S1_HW_SI_DIS 0x3200 #define B_S1_HW_SI_DIS_W_R_TRIG GENMASK(30, 28) #define R_P1_RXCK 0x32A0 @@ -5218,9 +7442,9 @@ #define R_SEG0R_PD_V2 0x6A74 #define R_SEG0R_EDCCA_LVL 0x4840 #define R_SEG0R_EDCCA_LVL_V1 0x4884 -#define B_SEG0R_PPDU_LVL_MSK GENMASK(31, 24) -#define B_SEG0R_EDCCA_LVL_P_MSK GENMASK(15, 8) -#define B_SEG0R_EDCCA_LVL_A_MSK GENMASK(7, 0) +#define B_EDCCA_LVL_MSK3 GENMASK(31, 24) +#define B_EDCCA_LVL_MSK1 GENMASK(15, 8) +#define B_EDCCA_LVL_MSK0 GENMASK(7, 0) #define B_SEG0R_PD_SPATIAL_REUSE_EN_MSK_V1 BIT(30) #define B_SEG0R_PD_SPATIAL_REUSE_EN_MSK BIT(29) #define B_SEG0R_PD_LOWER_BOUND_MSK GENMASK(10, 6) @@ -5476,6 +7700,10 @@ #define B_DCFO_WEIGHT_MSK_V1 GENMASK(31, 28) #define R_DCFO_OPT_V1 0x6260 #define B_DCFO_OPT_EN_V1 BIT(17) +#define R_SEG0R_EDCCA_LVL_BE 0x69EC +#define R_SEG0R_PPDU_LVL_BE 0x69F0 +#define R_SEGSND 0x6A14 +#define B_SEGSND_EN BIT(31) #define R_RPL_BIAS_COMP1 0x6DF0 #define B_RPL_BIAS_COMP1_MASK GENMASK(7, 0) #define R_P1_TSSI_ALIM1 0x7630 diff --git a/drivers/net/wireless/realtek/rtw89/rtw8851b.c b/drivers/net/wireless/realtek/rtw89/rtw8851b.c index dd15b904cd2fbe..5c167a9278ce4b 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8851b.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8851b.c @@ -205,6 +205,20 @@ static const struct rtw89_dig_regs rtw8851b_dig_regs = { B_PATH1_S20_FOLLOW_BY_PAGCUGC_EN_MSK}, }; +static const struct rtw89_edcca_regs rtw8851b_edcca_regs = { + .edcca_level = R_SEG0R_EDCCA_LVL_V1, + .edcca_mask = B_EDCCA_LVL_MSK0, + .edcca_p_mask = B_EDCCA_LVL_MSK1, + .ppdu_level = R_SEG0R_EDCCA_LVL_V1, + .ppdu_mask = B_EDCCA_LVL_MSK3, + .rpt_a = R_EDCCA_RPT_A, + .rpt_b = R_EDCCA_RPT_B, + .rpt_sel = R_EDCCA_RPT_SEL, + .rpt_sel_mask = B_EDCCA_RPT_SEL_MSK, + .tx_collision_t2r_st = R_TX_COLLISION_T2R_ST, + .tx_collision_t2r_st_mask = B_TX_COLLISION_T2R_ST_M, +}; + static const struct rtw89_btc_rf_trx_para rtw89_btc_8851b_rf_ul[] = { {255, 0, 0, 7}, /* 0 -> original */ {255, 2, 0, 7}, /* 1 -> for BT-connected ACI issue && BTG co-rx */ @@ -2360,8 +2374,8 @@ const struct rtw89_chip_info rtw8851b_chip_info = { .rsvd_ple_ofst = 0x2f800, .hfc_param_ini = rtw8851b_hfc_param_ini_pcie, .dle_mem = rtw8851b_dle_mem_pcie, - .wde_qempty_acq_num = 4, - .wde_qempty_mgq_sel = 4, + .wde_qempty_acq_grpnum = 4, + .wde_qempty_mgq_grpsel = 4, .rf_base_addr = {0xe000}, .pwr_on_seq = NULL, .pwr_off_seq = NULL, @@ -2440,13 +2454,15 @@ const struct rtw89_chip_info rtw8851b_chip_info = { .dcfo_comp = &rtw8851b_dcfo_comp, .dcfo_comp_sft = 12, .imr_info = &rtw8851b_imr_info, + .imr_dmac_table = NULL, + .imr_cmac_table = NULL, .rrsr_cfgs = &rtw8851b_rrsr_cfgs, .bss_clr_vld = {R_BSS_CLR_MAP_V1, B_BSS_CLR_MAP_VLD0}, .bss_clr_map_reg = R_BSS_CLR_MAP_V1, .dma_ch_mask = BIT(RTW89_DMA_ACH4) | BIT(RTW89_DMA_ACH5) | BIT(RTW89_DMA_ACH6) | BIT(RTW89_DMA_ACH7) | BIT(RTW89_DMA_B1MG) | BIT(RTW89_DMA_B1HI), - .edcca_lvl_reg = R_SEG0R_EDCCA_LVL_V1, + .edcca_regs = &rtw8851b_edcca_regs, #ifdef CONFIG_PM .wowlan_stub = &rtw_wowlan_stub_8851b, #endif diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852a.c b/drivers/net/wireless/realtek/rtw89/rtw8852a.c index 2bddd0acb19562..0c76c52ce22c71 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8852a.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8852a.c @@ -498,6 +498,20 @@ static const struct rtw89_dig_regs rtw8852a_dig_regs = { B_PATH1_S20_FOLLOW_BY_PAGCUGC_EN_MSK}, }; +static const struct rtw89_edcca_regs rtw8852a_edcca_regs = { + .edcca_level = R_SEG0R_EDCCA_LVL, + .edcca_mask = B_EDCCA_LVL_MSK0, + .edcca_p_mask = B_EDCCA_LVL_MSK1, + .ppdu_level = R_SEG0R_EDCCA_LVL, + .ppdu_mask = B_EDCCA_LVL_MSK3, + .rpt_a = R_EDCCA_RPT_A, + .rpt_b = R_EDCCA_RPT_B, + .rpt_sel = R_EDCCA_RPT_SEL, + .rpt_sel_mask = B_EDCCA_RPT_SEL_MSK, + .tx_collision_t2r_st = R_TX_COLLISION_T2R_ST, + .tx_collision_t2r_st_mask = B_TX_COLLISION_T2R_ST_M, +}; + static void rtw8852ae_efuse_parsing(struct rtw89_efuse *efuse, struct rtw8852a_efuse *map) { @@ -2095,8 +2109,8 @@ const struct rtw89_chip_info rtw8852a_chip_info = { .rsvd_ple_ofst = 0x6f800, .hfc_param_ini = rtw8852a_hfc_param_ini_pcie, .dle_mem = rtw8852a_dle_mem_pcie, - .wde_qempty_acq_num = 16, - .wde_qempty_mgq_sel = 16, + .wde_qempty_acq_grpnum = 16, + .wde_qempty_mgq_grpsel = 16, .rf_base_addr = {0xc000, 0xd000}, .pwr_on_seq = pwr_on_seq_8852a, .pwr_off_seq = pwr_off_seq_8852a, @@ -2177,11 +2191,13 @@ const struct rtw89_chip_info rtw8852a_chip_info = { .dcfo_comp = &rtw8852a_dcfo_comp, .dcfo_comp_sft = 10, .imr_info = &rtw8852a_imr_info, + .imr_dmac_table = NULL, + .imr_cmac_table = NULL, .rrsr_cfgs = &rtw8852a_rrsr_cfgs, .bss_clr_vld = {R_BSS_CLR_MAP, B_BSS_CLR_MAP_VLD0}, .bss_clr_map_reg = R_BSS_CLR_MAP, .dma_ch_mask = 0, - .edcca_lvl_reg = R_SEG0R_EDCCA_LVL, + .edcca_regs = &rtw8852a_edcca_regs, #ifdef CONFIG_PM .wowlan_stub = &rtw_wowlan_stub_8852a, #endif diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852b.c b/drivers/net/wireless/realtek/rtw89/rtw8852b.c index a576e4f478809b..de887a35f3fb4b 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8852b.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8852b.c @@ -330,6 +330,20 @@ static const struct rtw89_dig_regs rtw8852b_dig_regs = { B_PATH1_S20_FOLLOW_BY_PAGCUGC_EN_MSK}, }; +static const struct rtw89_edcca_regs rtw8852b_edcca_regs = { + .edcca_level = R_SEG0R_EDCCA_LVL_V1, + .edcca_mask = B_EDCCA_LVL_MSK0, + .edcca_p_mask = B_EDCCA_LVL_MSK1, + .ppdu_level = R_SEG0R_EDCCA_LVL_V1, + .ppdu_mask = B_EDCCA_LVL_MSK3, + .rpt_a = R_EDCCA_RPT_A, + .rpt_b = R_EDCCA_RPT_B, + .rpt_sel = R_EDCCA_RPT_SEL, + .rpt_sel_mask = B_EDCCA_RPT_SEL_MSK, + .tx_collision_t2r_st = R_TX_COLLISION_T2R_ST, + .tx_collision_t2r_st_mask = B_TX_COLLISION_T2R_ST_M, +}; + static const struct rtw89_btc_rf_trx_para rtw89_btc_8852b_rf_ul[] = { {255, 0, 0, 7}, /* 0 -> original */ {255, 2, 0, 7}, /* 1 -> for BT-connected ACI issue && BTG co-rx */ @@ -2529,8 +2543,8 @@ const struct rtw89_chip_info rtw8852b_chip_info = { .rsvd_ple_ofst = 0x2f800, .hfc_param_ini = rtw8852b_hfc_param_ini_pcie, .dle_mem = rtw8852b_dle_mem_pcie, - .wde_qempty_acq_num = 4, - .wde_qempty_mgq_sel = 4, + .wde_qempty_acq_grpnum = 4, + .wde_qempty_mgq_grpsel = 4, .rf_base_addr = {0xe000, 0xf000}, .pwr_on_seq = NULL, .pwr_off_seq = NULL, @@ -2611,13 +2625,15 @@ const struct rtw89_chip_info rtw8852b_chip_info = { .dcfo_comp = &rtw8852b_dcfo_comp, .dcfo_comp_sft = 10, .imr_info = &rtw8852b_imr_info, + .imr_dmac_table = NULL, + .imr_cmac_table = NULL, .rrsr_cfgs = &rtw8852b_rrsr_cfgs, .bss_clr_vld = {R_BSS_CLR_MAP_V1, B_BSS_CLR_MAP_VLD0}, .bss_clr_map_reg = R_BSS_CLR_MAP_V1, .dma_ch_mask = BIT(RTW89_DMA_ACH4) | BIT(RTW89_DMA_ACH5) | BIT(RTW89_DMA_ACH6) | BIT(RTW89_DMA_ACH7) | BIT(RTW89_DMA_B1MG) | BIT(RTW89_DMA_B1HI), - .edcca_lvl_reg = R_SEG0R_EDCCA_LVL_V1, + .edcca_regs = &rtw8852b_edcca_regs, #ifdef CONFIG_PM .wowlan_stub = &rtw_wowlan_stub_8852b, #endif diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852c.c b/drivers/net/wireless/realtek/rtw89/rtw8852c.c index ea152a4613f219..8618d0204f665e 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8852c.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8852c.c @@ -167,6 +167,20 @@ static const struct rtw89_dig_regs rtw8852c_dig_regs = { B_PATH1_S20_FOLLOW_BY_PAGCUGC_EN_MSK}, }; +static const struct rtw89_edcca_regs rtw8852c_edcca_regs = { + .edcca_level = R_SEG0R_EDCCA_LVL, + .edcca_mask = B_EDCCA_LVL_MSK0, + .edcca_p_mask = B_EDCCA_LVL_MSK1, + .ppdu_level = R_SEG0R_EDCCA_LVL, + .ppdu_mask = B_EDCCA_LVL_MSK3, + .rpt_a = R_EDCCA_RPT_A, + .rpt_b = R_EDCCA_RPT_B, + .rpt_sel = R_EDCCA_RPT_SEL, + .rpt_sel_mask = B_EDCCA_RPT_SEL_MSK, + .tx_collision_t2r_st = R_TX_COLLISION_T2R_ST, + .tx_collision_t2r_st_mask = B_TX_COLLISION_T2R_ST_M, +}; + static void rtw8852c_ctrl_btg_bt_rx(struct rtw89_dev *rtwdev, bool en, enum rtw89_phy_idx phy_idx); @@ -2865,8 +2879,8 @@ const struct rtw89_chip_info rtw8852c_chip_info = { .rsvd_ple_ofst = 0x6f800, .hfc_param_ini = rtw8852c_hfc_param_ini_pcie, .dle_mem = rtw8852c_dle_mem_pcie, - .wde_qempty_acq_num = 16, - .wde_qempty_mgq_sel = 16, + .wde_qempty_acq_grpnum = 16, + .wde_qempty_mgq_grpsel = 16, .rf_base_addr = {0xe000, 0xf000}, .pwr_on_seq = NULL, .pwr_off_seq = NULL, @@ -2950,11 +2964,13 @@ const struct rtw89_chip_info rtw8852c_chip_info = { .dcfo_comp = &rtw8852c_dcfo_comp, .dcfo_comp_sft = 12, .imr_info = &rtw8852c_imr_info, + .imr_dmac_table = NULL, + .imr_cmac_table = NULL, .rrsr_cfgs = &rtw8852c_rrsr_cfgs, .bss_clr_vld = {R_BSS_CLR_MAP, B_BSS_CLR_MAP_VLD0}, .bss_clr_map_reg = R_BSS_CLR_MAP, .dma_ch_mask = 0, - .edcca_lvl_reg = R_SEG0R_EDCCA_LVL, + .edcca_regs = &rtw8852c_edcca_regs, #ifdef CONFIG_PM .wowlan_stub = &rtw_wowlan_stub_8852c, #endif diff --git a/drivers/net/wireless/realtek/rtw89/rtw8922a.c b/drivers/net/wireless/realtek/rtw89/rtw8922a.c index d190f095a5a8f4..0e7300cc6d9e54 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8922a.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8922a.c @@ -15,6 +15,110 @@ #define RTW8922A_MODULE_FIRMWARE \ RTW8922A_FW_BASENAME ".bin" +static const struct rtw89_hfc_ch_cfg rtw8922a_hfc_chcfg_pcie[] = { + {2, 1641, grp_0}, /* ACH 0 */ + {2, 1641, grp_0}, /* ACH 1 */ + {2, 1641, grp_0}, /* ACH 2 */ + {2, 1641, grp_0}, /* ACH 3 */ + {2, 1641, grp_1}, /* ACH 4 */ + {2, 1641, grp_1}, /* ACH 5 */ + {2, 1641, grp_1}, /* ACH 6 */ + {2, 1641, grp_1}, /* ACH 7 */ + {2, 1641, grp_0}, /* B0MGQ */ + {2, 1641, grp_0}, /* B0HIQ */ + {2, 1641, grp_1}, /* B1MGQ */ + {2, 1641, grp_1}, /* B1HIQ */ + {0, 0, 0}, /* FWCMDQ */ + {0, 0, 0}, /* BMC */ + {0, 0, 0}, /* H2D */ +}; + +static const struct rtw89_hfc_pub_cfg rtw8922a_hfc_pubcfg_pcie = { + 1651, /* Group 0 */ + 1651, /* Group 1 */ + 3302, /* Public Max */ + 0, /* WP threshold */ +}; + +static const struct rtw89_hfc_param_ini rtw8922a_hfc_param_ini_pcie[] = { + [RTW89_QTA_SCC] = {rtw8922a_hfc_chcfg_pcie, &rtw8922a_hfc_pubcfg_pcie, + &rtw89_mac_size.hfc_prec_cfg_c0, RTW89_HCIFC_POH}, + [RTW89_QTA_DLFW] = {NULL, NULL, &rtw89_mac_size.hfc_prec_cfg_c2, + RTW89_HCIFC_POH}, + [RTW89_QTA_INVALID] = {NULL}, +}; + +static const struct rtw89_dle_mem rtw8922a_dle_mem_pcie[] = { + [RTW89_QTA_SCC] = {RTW89_QTA_SCC, &rtw89_mac_size.wde_size0_v1, + &rtw89_mac_size.ple_size0_v1, &rtw89_mac_size.wde_qt0_v1, + &rtw89_mac_size.wde_qt0_v1, &rtw89_mac_size.ple_qt0, + &rtw89_mac_size.ple_qt1, &rtw89_mac_size.ple_rsvd_qt0, + &rtw89_mac_size.rsvd0_size0, &rtw89_mac_size.rsvd1_size0}, + [RTW89_QTA_DLFW] = {RTW89_QTA_DLFW, &rtw89_mac_size.wde_size4_v1, + &rtw89_mac_size.ple_size3_v1, &rtw89_mac_size.wde_qt4, + &rtw89_mac_size.wde_qt4, &rtw89_mac_size.ple_qt9, + &rtw89_mac_size.ple_qt9, &rtw89_mac_size.ple_rsvd_qt1, + &rtw89_mac_size.rsvd0_size0, &rtw89_mac_size.rsvd1_size0}, + [RTW89_QTA_INVALID] = {RTW89_QTA_INVALID, NULL, NULL, NULL, NULL, NULL, + NULL}, +}; + +static const struct rtw89_reg_imr rtw8922a_imr_dmac_regs[] = { + {R_BE_DISP_HOST_IMR, B_BE_DISP_HOST_IMR_CLR, B_BE_DISP_HOST_IMR_SET}, + {R_BE_DISP_CPU_IMR, B_BE_DISP_CPU_IMR_CLR, B_BE_DISP_CPU_IMR_SET}, + {R_BE_DISP_OTHER_IMR, B_BE_DISP_OTHER_IMR_CLR, B_BE_DISP_OTHER_IMR_SET}, + {R_BE_PKTIN_ERR_IMR, B_BE_PKTIN_ERR_IMR_CLR, B_BE_PKTIN_ERR_IMR_SET}, + {R_BE_INTERRUPT_MASK_REG, B_BE_INTERRUPT_MASK_REG_CLR, B_BE_INTERRUPT_MASK_REG_SET}, + {R_BE_MLO_ERR_IDCT_IMR, B_BE_MLO_ERR_IDCT_IMR_CLR, B_BE_MLO_ERR_IDCT_IMR_SET}, + {R_BE_MPDU_TX_ERR_IMR, B_BE_MPDU_TX_ERR_IMR_CLR, B_BE_MPDU_TX_ERR_IMR_SET}, + {R_BE_MPDU_RX_ERR_IMR, B_BE_MPDU_RX_ERR_IMR_CLR, B_BE_MPDU_RX_ERR_IMR_SET}, + {R_BE_SEC_ERROR_IMR, B_BE_SEC_ERROR_IMR_CLR, B_BE_SEC_ERROR_IMR_SET}, + {R_BE_CPUIO_ERR_IMR, B_BE_CPUIO_ERR_IMR_CLR, B_BE_CPUIO_ERR_IMR_SET}, + {R_BE_WDE_ERR_IMR, B_BE_WDE_ERR_IMR_CLR, B_BE_WDE_ERR_IMR_SET}, + {R_BE_WDE_ERR1_IMR, B_BE_WDE_ERR1_IMR_CLR, B_BE_WDE_ERR1_IMR_SET}, + {R_BE_PLE_ERR_IMR, B_BE_PLE_ERR_IMR_CLR, B_BE_PLE_ERR_IMR_SET}, + {R_BE_PLE_ERRFLAG1_IMR, B_BE_PLE_ERRFLAG1_IMR_CLR, B_BE_PLE_ERRFLAG1_IMR_SET}, + {R_BE_WDRLS_ERR_IMR, B_BE_WDRLS_ERR_IMR_CLR, B_BE_WDRLS_ERR_IMR_SET}, + {R_BE_TXPKTCTL_B0_ERRFLAG_IMR, B_BE_TXPKTCTL_B0_ERRFLAG_IMR_CLR, + B_BE_TXPKTCTL_B0_ERRFLAG_IMR_SET}, + {R_BE_TXPKTCTL_B1_ERRFLAG_IMR, B_BE_TXPKTCTL_B1_ERRFLAG_IMR_CLR, + B_BE_TXPKTCTL_B1_ERRFLAG_IMR_SET}, + {R_BE_BBRPT_COM_ERR_IMR, B_BE_BBRPT_COM_ERR_IMR_CLR, B_BE_BBRPT_COM_ERR_IMR_SET}, + {R_BE_BBRPT_CHINFO_ERR_IMR, B_BE_BBRPT_CHINFO_ERR_IMR_CLR, + B_BE_BBRPT_CHINFO_ERR_IMR_SET}, + {R_BE_BBRPT_DFS_ERR_IMR, B_BE_BBRPT_DFS_ERR_IMR_CLR, B_BE_BBRPT_DFS_ERR_IMR_SET}, + {R_BE_LA_ERRFLAG_IMR, B_BE_LA_ERRFLAG_IMR_CLR, B_BE_LA_ERRFLAG_IMR_SET}, + {R_BE_CH_INFO_DBGFLAG_IMR, B_BE_CH_INFO_DBGFLAG_IMR_CLR, B_BE_CH_INFO_DBGFLAG_IMR_SET}, + {R_BE_PLRLS_ERR_IMR, B_BE_PLRLS_ERR_IMR_CLR, B_BE_PLRLS_ERR_IMR_SET}, + {R_BE_HAXI_IDCT_MSK, B_BE_HAXI_IDCT_MSK_CLR, B_BE_HAXI_IDCT_MSK_SET}, +}; + +static const struct rtw89_imr_table rtw8922a_imr_dmac_table = { + .regs = rtw8922a_imr_dmac_regs, + .n_regs = ARRAY_SIZE(rtw8922a_imr_dmac_regs), +}; + +static const struct rtw89_reg_imr rtw8922a_imr_cmac_regs[] = { + {R_BE_RESP_IMR, B_BE_RESP_IMR_CLR, B_BE_RESP_IMR_SET}, + {R_BE_RX_ERROR_FLAG_IMR, B_BE_RX_ERROR_FLAG_IMR_CLR, B_BE_RX_ERROR_FLAG_IMR_SET}, + {R_BE_TX_ERROR_FLAG_IMR, B_BE_TX_ERROR_FLAG_IMR_CLR, B_BE_TX_ERROR_FLAG_IMR_SET}, + {R_BE_RX_ERROR_FLAG_IMR_1, B_BE_TX_ERROR_FLAG_IMR_1_CLR, B_BE_TX_ERROR_FLAG_IMR_1_SET}, + {R_BE_PTCL_IMR1, B_BE_PTCL_IMR1_CLR, B_BE_PTCL_IMR1_SET}, + {R_BE_PTCL_IMR0, B_BE_PTCL_IMR0_CLR, B_BE_PTCL_IMR0_SET}, + {R_BE_PTCL_IMR_2, B_BE_PTCL_IMR_2_CLR, B_BE_PTCL_IMR_2_SET}, + {R_BE_SCHEDULE_ERR_IMR, B_BE_SCHEDULE_ERR_IMR_CLR, B_BE_SCHEDULE_ERR_IMR_SET}, + {R_BE_C0_TXPWR_IMR, B_BE_C0_TXPWR_IMR_CLR, B_BE_C0_TXPWR_IMR_SET}, + {R_BE_TRXPTCL_ERROR_INDICA_MASK, B_BE_TRXPTCL_ERROR_INDICA_MASK_CLR, + B_BE_TRXPTCL_ERROR_INDICA_MASK_SET}, + {R_BE_RX_ERR_IMR, B_BE_RX_ERR_IMR_CLR, B_BE_RX_ERR_IMR_SET}, + {R_BE_PHYINFO_ERR_IMR_V1, B_BE_PHYINFO_ERR_IMR_V1_CLR, B_BE_PHYINFO_ERR_IMR_V1_SET}, +}; + +static const struct rtw89_imr_table rtw8922a_imr_cmac_table = { + .regs = rtw8922a_imr_cmac_regs, + .n_regs = ARRAY_SIZE(rtw8922a_imr_cmac_regs), +}; + static const struct rtw89_efuse_block_cfg rtw8922a_efuse_blocks[] = { [RTW89_EFUSE_BLOCK_SYS] = {.offset = 0x00000, .size = 0x310}, [RTW89_EFUSE_BLOCK_RF] = {.offset = 0x10000, .size = 0x240}, @@ -26,6 +130,241 @@ static const struct rtw89_efuse_block_cfg rtw8922a_efuse_blocks[] = { [RTW89_EFUSE_BLOCK_ADIE] = {.offset = 0x70000, .size = 0x10}, }; +static int rtw8922a_pwr_on_func(struct rtw89_dev *rtwdev) +{ + struct rtw89_hal *hal = &rtwdev->hal; + u32 val32; + int ret; + + rtw89_write32_clr(rtwdev, R_BE_SYS_PW_CTRL, B_BE_AFSM_WLSUS_EN | + B_BE_AFSM_PCIE_SUS_EN); + rtw89_write32_set(rtwdev, R_BE_SYS_PW_CTRL, B_BE_DIS_WLBT_PDNSUSEN_SOPC); + rtw89_write32_set(rtwdev, R_BE_WLLPS_CTRL, B_BE_DIS_WLBT_LPSEN_LOPC); + rtw89_write32_clr(rtwdev, R_BE_SYS_PW_CTRL, B_BE_APDM_HPDN); + rtw89_write32_clr(rtwdev, R_BE_SYS_PW_CTRL, B_BE_APFM_SWLPS); + + ret = read_poll_timeout(rtw89_read32, val32, val32 & B_BE_RDY_SYSPWR, + 1000, 3000000, false, rtwdev, R_BE_SYS_PW_CTRL); + if (ret) + return ret; + + rtw89_write32_set(rtwdev, R_BE_SYS_PW_CTRL, B_BE_EN_WLON); + rtw89_write32_set(rtwdev, R_BE_WLRESUME_CTRL, B_BE_LPSROP_CMAC0 | + B_BE_LPSROP_CMAC1); + rtw89_write32_set(rtwdev, R_BE_SYS_PW_CTRL, B_BE_APFN_ONMAC); + + ret = read_poll_timeout(rtw89_read32, val32, !(val32 & B_BE_APFN_ONMAC), + 1000, 3000000, false, rtwdev, R_BE_SYS_PW_CTRL); + if (ret) + return ret; + + rtw89_write32_clr(rtwdev, R_BE_AFE_ON_CTRL1, B_BE_REG_CK_MON_CK960M_EN); + rtw89_write8_set(rtwdev, R_BE_ANAPAR_POW_MAC, B_BE_POW_PC_LDO_PORT0 | + B_BE_POW_PC_LDO_PORT1); + rtw89_write32_clr(rtwdev, R_BE_FEN_RST_ENABLE, B_BE_R_SYM_ISO_ADDA_P02PP | + B_BE_R_SYM_ISO_ADDA_P12PP); + rtw89_write8_set(rtwdev, R_BE_PLATFORM_ENABLE, B_BE_PLATFORM_EN); + rtw89_write32_set(rtwdev, R_BE_HCI_OPT_CTRL, B_BE_HAXIDMA_IO_EN); + + ret = read_poll_timeout(rtw89_read32, val32, val32 & B_BE_HAXIDMA_IO_ST, + 1000, 3000000, false, rtwdev, R_BE_HCI_OPT_CTRL); + if (ret) + return ret; + + ret = read_poll_timeout(rtw89_read32, val32, !(val32 & B_BE_HAXIDMA_BACKUP_RESTORE_ST), + 1000, 3000000, false, rtwdev, R_BE_HCI_OPT_CTRL); + if (ret) + return ret; + + rtw89_write32_set(rtwdev, R_BE_HCI_OPT_CTRL, B_BE_HCI_WLAN_IO_EN); + + ret = read_poll_timeout(rtw89_read32, val32, val32 & B_BE_HCI_WLAN_IO_ST, + 1000, 3000000, false, rtwdev, R_BE_HCI_OPT_CTRL); + if (ret) + return ret; + + rtw89_write32_clr(rtwdev, R_BE_SYS_SDIO_CTRL, B_BE_PCIE_FORCE_IBX_EN); + + ret = rtw89_mac_write_xtal_si(rtwdev, XTAL_SI_PLL, 0x02, 0x02); + if (ret) + return ret; + ret = rtw89_mac_write_xtal_si(rtwdev, XTAL_SI_PLL, 0x01, 0x01); + if (ret) + return ret; + + rtw89_write32_set(rtwdev, R_BE_SYS_ADIE_PAD_PWR_CTRL, B_BE_SYM_PADPDN_WL_RFC1_1P3); + + ret = rtw89_mac_write_xtal_si(rtwdev, XTAL_SI_ANAPAR_WL, 0x40, 0x40); + if (ret) + return ret; + + rtw89_write32_set(rtwdev, R_BE_SYS_ADIE_PAD_PWR_CTRL, B_BE_SYM_PADPDN_WL_RFC0_1P3); + + ret = rtw89_mac_write_xtal_si(rtwdev, XTAL_SI_ANAPAR_WL, 0x20, 0x20); + if (ret) + return ret; + ret = rtw89_mac_write_xtal_si(rtwdev, XTAL_SI_ANAPAR_WL, 0x04, 0x04); + if (ret) + return ret; + ret = rtw89_mac_write_xtal_si(rtwdev, XTAL_SI_ANAPAR_WL, 0x08, 0x08); + if (ret) + return ret; + ret = rtw89_mac_write_xtal_si(rtwdev, XTAL_SI_ANAPAR_WL, 0, 0x10); + if (ret) + return ret; + ret = rtw89_mac_write_xtal_si(rtwdev, XTAL_SI_WL_RFC_S0, 0xEB, 0xFF); + if (ret) + return ret; + ret = rtw89_mac_write_xtal_si(rtwdev, XTAL_SI_WL_RFC_S1, 0xEB, 0xFF); + if (ret) + return ret; + ret = rtw89_mac_write_xtal_si(rtwdev, XTAL_SI_ANAPAR_WL, 0x01, 0x01); + if (ret) + return ret; + ret = rtw89_mac_write_xtal_si(rtwdev, XTAL_SI_ANAPAR_WL, 0x02, 0x02); + if (ret) + return ret; + ret = rtw89_mac_write_xtal_si(rtwdev, XTAL_SI_ANAPAR_WL, 0, 0x80); + if (ret) + return ret; + ret = rtw89_mac_write_xtal_si(rtwdev, XTAL_SI_XREF_RF1, 0, 0x40); + if (ret) + return ret; + ret = rtw89_mac_write_xtal_si(rtwdev, XTAL_SI_XREF_RF2, 0, 0x40); + if (ret) + return ret; + ret = rtw89_mac_write_xtal_si(rtwdev, XTAL_SI_PLL_1, 0x40, 0x60); + if (ret) + return ret; + + if (hal->cv != CHIP_CAV) { + rtw89_write32_set(rtwdev, R_BE_PMC_DBG_CTRL2, B_BE_SYSON_DIS_PMCR_BE_WRMSK); + rtw89_write32_set(rtwdev, R_BE_SYS_ISO_CTRL, B_BE_ISO_EB2CORE); + rtw89_write32_clr(rtwdev, R_BE_SYS_ISO_CTRL, B_BE_PWC_EV2EF_B); + + mdelay(1); + + rtw89_write32_clr(rtwdev, R_BE_SYS_ISO_CTRL, B_BE_PWC_EV2EF_S); + rtw89_write32_clr(rtwdev, R_BE_PMC_DBG_CTRL2, B_BE_SYSON_DIS_PMCR_BE_WRMSK); + } + + rtw89_write32_set(rtwdev, R_BE_DMAC_FUNC_EN, + B_BE_MAC_FUNC_EN | B_BE_DMAC_FUNC_EN | B_BE_MPDU_PROC_EN | + B_BE_WD_RLS_EN | B_BE_DLE_WDE_EN | B_BE_TXPKT_CTRL_EN | + B_BE_STA_SCH_EN | B_BE_DLE_PLE_EN | B_BE_PKT_BUF_EN | + B_BE_DMAC_TBL_EN | B_BE_PKT_IN_EN | B_BE_DLE_CPUIO_EN | + B_BE_DISPATCHER_EN | B_BE_BBRPT_EN | B_BE_MAC_SEC_EN | + B_BE_H_AXIDMA_EN | B_BE_DMAC_MLO_EN | B_BE_PLRLS_EN | + B_BE_P_AXIDMA_EN | B_BE_DLE_DATACPUIO_EN | B_BE_LTR_CTL_EN); + + set_bit(RTW89_FLAG_DMAC_FUNC, rtwdev->flags); + + rtw89_write32_set(rtwdev, R_BE_CMAC_SHARE_FUNC_EN, + B_BE_CMAC_SHARE_EN | B_BE_RESPBA_EN | B_BE_ADDRSRCH_EN | + B_BE_BTCOEX_EN); + rtw89_write32_set(rtwdev, R_BE_CMAC_FUNC_EN, + B_BE_CMAC_EN | B_BE_CMAC_TXEN | B_BE_CMAC_RXEN | + B_BE_SIGB_EN | B_BE_PHYINTF_EN | B_BE_CMAC_DMA_EN | + B_BE_PTCLTOP_EN | B_BE_SCHEDULER_EN | B_BE_TMAC_EN | + B_BE_RMAC_EN | B_BE_TXTIME_EN | B_BE_RESP_PKTCTL_EN); + + set_bit(RTW89_FLAG_CMAC0_FUNC, rtwdev->flags); + + rtw89_write32_set(rtwdev, R_BE_FEN_RST_ENABLE, B_BE_FEN_BB_IP_RSTN | + B_BE_FEN_BBPLAT_RSTB); + + return 0; +} + +static int rtw8922a_pwr_off_func(struct rtw89_dev *rtwdev) +{ + u32 val32; + int ret; + + ret = rtw89_mac_write_xtal_si(rtwdev, XTAL_SI_ANAPAR_WL, 0x10, 0x10); + if (ret) + return ret; + ret = rtw89_mac_write_xtal_si(rtwdev, XTAL_SI_ANAPAR_WL, 0, 0x08); + if (ret) + return ret; + ret = rtw89_mac_write_xtal_si(rtwdev, XTAL_SI_ANAPAR_WL, 0, 0x04); + if (ret) + return ret; + ret = rtw89_mac_write_xtal_si(rtwdev, XTAL_SI_WL_RFC_S0, 0xC6, 0xFF); + if (ret) + return ret; + ret = rtw89_mac_write_xtal_si(rtwdev, XTAL_SI_WL_RFC_S1, 0xC6, 0xFF); + if (ret) + return ret; + ret = rtw89_mac_write_xtal_si(rtwdev, XTAL_SI_ANAPAR_WL, 0x80, 0x80); + if (ret) + return ret; + ret = rtw89_mac_write_xtal_si(rtwdev, XTAL_SI_ANAPAR_WL, 0, 0x02); + if (ret) + return ret; + ret = rtw89_mac_write_xtal_si(rtwdev, XTAL_SI_ANAPAR_WL, 0, 0x01); + if (ret) + return ret; + ret = rtw89_mac_write_xtal_si(rtwdev, XTAL_SI_PLL, 0x02, 0xFF); + if (ret) + return ret; + ret = rtw89_mac_write_xtal_si(rtwdev, XTAL_SI_PLL, 0x00, 0xFF); + if (ret) + return ret; + + rtw89_write32_set(rtwdev, R_BE_FEN_RST_ENABLE, B_BE_R_SYM_ISO_ADDA_P02PP | + B_BE_R_SYM_ISO_ADDA_P12PP); + rtw89_write8_clr(rtwdev, R_BE_ANAPAR_POW_MAC, B_BE_POW_PC_LDO_PORT0 | + B_BE_POW_PC_LDO_PORT1); + rtw89_write32_set(rtwdev, R_BE_SYS_PW_CTRL, B_BE_EN_WLON); + rtw89_write8_clr(rtwdev, R_BE_FEN_RST_ENABLE, B_BE_FEN_BB_IP_RSTN | + B_BE_FEN_BBPLAT_RSTB); + rtw89_write32_clr(rtwdev, R_BE_SYS_ADIE_PAD_PWR_CTRL, B_BE_SYM_PADPDN_WL_RFC0_1P3); + + ret = rtw89_mac_write_xtal_si(rtwdev, XTAL_SI_ANAPAR_WL, 0, 0x20); + if (ret) + return ret; + + rtw89_write32_clr(rtwdev, R_BE_SYS_ADIE_PAD_PWR_CTRL, B_BE_SYM_PADPDN_WL_RFC1_1P3); + + ret = rtw89_mac_write_xtal_si(rtwdev, XTAL_SI_ANAPAR_WL, 0, 0x40); + if (ret) + return ret; + + rtw89_write32_clr(rtwdev, R_BE_HCI_OPT_CTRL, B_BE_HAXIDMA_IO_EN); + + ret = read_poll_timeout(rtw89_read32, val32, !(val32 & B_BE_HAXIDMA_IO_ST), + 1000, 3000000, false, rtwdev, R_BE_HCI_OPT_CTRL); + if (ret) + return ret; + + ret = read_poll_timeout(rtw89_read32, val32, !(val32 & B_BE_HAXIDMA_BACKUP_RESTORE_ST), + 1000, 3000000, false, rtwdev, R_BE_HCI_OPT_CTRL); + if (ret) + return ret; + + rtw89_write32_clr(rtwdev, R_BE_HCI_OPT_CTRL, B_BE_HCI_WLAN_IO_EN); + + ret = read_poll_timeout(rtw89_read32, val32, !(val32 & B_BE_HCI_WLAN_IO_ST), + 1000, 3000000, false, rtwdev, R_BE_HCI_OPT_CTRL); + if (ret) + return ret; + + rtw89_write32_set(rtwdev, R_BE_SYS_PW_CTRL, B_BE_APFM_OFFMAC); + + ret = read_poll_timeout(rtw89_read32, val32, !(val32 & B_BE_APFM_OFFMAC), + 1000, 3000000, false, rtwdev, R_BE_SYS_PW_CTRL); + if (ret) + return ret; + + rtw89_write32(rtwdev, R_BE_WLLPS_CTRL, 0x0000A1B2); + rtw89_write32_set(rtwdev, R_BE_SYS_PW_CTRL, B_BE_XTAL_OFF_A_DIE); + rtw89_write32_set(rtwdev, R_BE_SYS_PW_CTRL, B_BE_APFM_SWLPS); + rtw89_write32(rtwdev, R_BE_UDM1, 0); + + return 0; +} + static void rtw8922a_efuse_parsing_tssi(struct rtw89_dev *rtwdev, struct rtw8922a_efuse *map) { @@ -273,6 +612,8 @@ static const struct wiphy_wowlan_support rtw_wowlan_stub_8922a = { static const struct rtw89_chip_ops rtw8922a_chip_ops = { .read_efuse = rtw8922a_read_efuse, .read_phycap = rtw8922a_read_phycap, + .pwr_on_func = rtw8922a_pwr_on_func, + .pwr_off_func = rtw8922a_pwr_off_func, }; const struct rtw89_chip_info rtw8922a_chip_info = { @@ -292,6 +633,10 @@ const struct rtw89_chip_info rtw8922a_chip_info = { .max_amsdu_limit = 8000, .dis_2g_40m_ul_ofdma = false, .rsvd_ple_ofst = 0x8f800, + .hfc_param_ini = rtw8922a_hfc_param_ini_pcie, + .dle_mem = rtw8922a_dle_mem_pcie, + .wde_qempty_acq_grpnum = 4, + .wde_qempty_mgq_grpsel = 4, .rf_base_addr = {0xe000, 0xf000}, .pwr_on_seq = NULL, .pwr_off_seq = NULL, @@ -347,6 +692,8 @@ const struct rtw89_chip_info rtw8922a_chip_info = { .dcfo_comp = NULL, .dcfo_comp_sft = 0, .imr_info = NULL, + .imr_dmac_table = &rtw8922a_imr_dmac_table, + .imr_cmac_table = &rtw8922a_imr_cmac_table, .bss_clr_vld = {R_BSS_CLR_VLD_V2, B_BSS_CLR_VLD0_V2}, .bss_clr_map_reg = R_BSS_CLR_MAP_V2, .dma_ch_mask = 0, diff --git a/drivers/net/wireless/realtek/rtw89/ser.c b/drivers/net/wireless/realtek/rtw89/ser.c index 1e4a79a3b814ad..99896d85d2f81e 100644 --- a/drivers/net/wireless/realtek/rtw89/ser.c +++ b/drivers/net/wireless/realtek/rtw89/ser.c @@ -590,6 +590,14 @@ struct __fw_backtrace_info { static_assert(RTW89_FW_BACKTRACE_INFO_SIZE == sizeof(struct __fw_backtrace_info)); +static u32 convert_addr_from_wcpu(u32 wcpu_addr) +{ + if (wcpu_addr < 0x30000000) + return wcpu_addr; + + return wcpu_addr & GENMASK(28, 0); +} + static int rtw89_ser_fw_backtrace_dump(struct rtw89_dev *rtwdev, u8 *buf, const struct __fw_backtrace_entry *ent) { @@ -597,7 +605,7 @@ static int rtw89_ser_fw_backtrace_dump(struct rtw89_dev *rtwdev, u8 *buf, const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def; u32 filter_model_addr = mac->filter_model_addr; u32 indir_access_addr = mac->indir_access_addr; - u32 fwbt_addr = ent->wcpu_addr & RTW89_WCPU_BASE_MASK; + u32 fwbt_addr = convert_addr_from_wcpu(ent->wcpu_addr); u32 fwbt_size = ent->size; u32 fwbt_key = ent->key; u32 i; diff --git a/drivers/net/wireless/realtek/rtw89/wow.c b/drivers/net/wireless/realtek/rtw89/wow.c index 660bf2ece9277c..5c7ca36c09b6bf 100644 --- a/drivers/net/wireless/realtek/rtw89/wow.c +++ b/drivers/net/wireless/realtek/rtw89/wow.c @@ -73,13 +73,14 @@ static int rtw89_wow_config_mac(struct rtw89_dev *rtwdev, bool enable_wow) static void rtw89_wow_set_rx_filter(struct rtw89_dev *rtwdev, bool enable) { + const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def; enum rtw89_mac_fwd_target fwd_target = enable ? RTW89_FWD_DONT_CARE : RTW89_FWD_TO_HOST; - rtw89_mac_typ_fltr_opt(rtwdev, RTW89_MGNT, fwd_target, RTW89_MAC_0); - rtw89_mac_typ_fltr_opt(rtwdev, RTW89_CTRL, fwd_target, RTW89_MAC_0); - rtw89_mac_typ_fltr_opt(rtwdev, RTW89_DATA, fwd_target, RTW89_MAC_0); + mac->typ_fltr_opt(rtwdev, RTW89_MGNT, fwd_target, RTW89_MAC_0); + mac->typ_fltr_opt(rtwdev, RTW89_CTRL, fwd_target, RTW89_MAC_0); + mac->typ_fltr_opt(rtwdev, RTW89_DATA, fwd_target, RTW89_MAC_0); } static void rtw89_wow_show_wakeup_reason(struct rtw89_dev *rtwdev) diff --git a/drivers/net/wireless/silabs/wfx/sta.c b/drivers/net/wireless/silabs/wfx/sta.c index 1b6c158457b429..537caf9d914a71 100644 --- a/drivers/net/wireless/silabs/wfx/sta.c +++ b/drivers/net/wireless/silabs/wfx/sta.c @@ -336,29 +336,38 @@ static int wfx_upload_ap_templates(struct wfx_vif *wvif) return 0; } -static void wfx_set_mfp_ap(struct wfx_vif *wvif) +static int wfx_set_mfp_ap(struct wfx_vif *wvif) { struct ieee80211_vif *vif = wvif_to_vif(wvif); struct sk_buff *skb = ieee80211_beacon_get(wvif->wdev->hw, vif, 0); const int ieoffset = offsetof(struct ieee80211_mgmt, u.beacon.variable); - const u16 *ptr = (u16 *)cfg80211_find_ie(WLAN_EID_RSN, skb->data + ieoffset, - skb->len - ieoffset); const int pairwise_cipher_suite_count_offset = 8 / sizeof(u16); const int pairwise_cipher_suite_size = 4 / sizeof(u16); const int akm_suite_size = 4 / sizeof(u16); + const u16 *ptr; - if (ptr) { - ptr += pairwise_cipher_suite_count_offset; - if (WARN_ON(ptr > (u16 *)skb_tail_pointer(skb))) - return; - ptr += 1 + pairwise_cipher_suite_size * *ptr; - if (WARN_ON(ptr > (u16 *)skb_tail_pointer(skb))) - return; - ptr += 1 + akm_suite_size * *ptr; - if (WARN_ON(ptr > (u16 *)skb_tail_pointer(skb))) - return; - wfx_hif_set_mfp(wvif, *ptr & BIT(7), *ptr & BIT(6)); - } + if (unlikely(!skb)) + return -ENOMEM; + + ptr = (u16 *)cfg80211_find_ie(WLAN_EID_RSN, skb->data + ieoffset, + skb->len - ieoffset); + if (unlikely(!ptr)) + return -EINVAL; + + ptr += pairwise_cipher_suite_count_offset; + if (WARN_ON(ptr > (u16 *)skb_tail_pointer(skb))) + return -EINVAL; + + ptr += 1 + pairwise_cipher_suite_size * *ptr; + if (WARN_ON(ptr > (u16 *)skb_tail_pointer(skb))) + return -EINVAL; + + ptr += 1 + akm_suite_size * *ptr; + if (WARN_ON(ptr > (u16 *)skb_tail_pointer(skb))) + return -EINVAL; + + wfx_hif_set_mfp(wvif, *ptr & BIT(7), *ptr & BIT(6)); + return 0; } int wfx_start_ap(struct ieee80211_hw *hw, struct ieee80211_vif *vif, @@ -376,8 +385,7 @@ int wfx_start_ap(struct ieee80211_hw *hw, struct ieee80211_vif *vif, ret = wfx_hif_start(wvif, &vif->bss_conf, wvif->channel); if (ret > 0) return -EIO; - wfx_set_mfp_ap(wvif); - return ret; + return wfx_set_mfp_ap(wvif); } void wfx_stop_ap(struct ieee80211_hw *hw, struct ieee80211_vif *vif, diff --git a/drivers/platform/x86/amd/Kconfig b/drivers/platform/x86/amd/Kconfig index 55f3a2fc6aec7e..54753213cc61c2 100644 --- a/drivers/platform/x86/amd/Kconfig +++ b/drivers/platform/x86/amd/Kconfig @@ -18,3 +18,17 @@ config AMD_HSMP If you choose to compile this driver as a module the module will be called amd_hsmp. + +config AMD_WBRF + bool "AMD Wifi RF Band mitigations (WBRF)" + depends on ACPI + help + WBRF(Wifi Band RFI mitigation) mechanism allows Wifi drivers + to notify the frequencies they are using so that other hardware + can be reconfigured to avoid harmonic conflicts. + + AMD provides an ACPI based mechanism to support WBRF on platform with + appropriate underlying support. + + This mechanism will only be activated on platforms that advertise a + need for it. diff --git a/drivers/platform/x86/amd/Makefile b/drivers/platform/x86/amd/Makefile index f04932b7a7d124..dcec0a46f8af16 100644 --- a/drivers/platform/x86/amd/Makefile +++ b/drivers/platform/x86/amd/Makefile @@ -8,3 +8,4 @@ obj-$(CONFIG_AMD_PMC) += pmc/ amd_hsmp-y := hsmp.o obj-$(CONFIG_AMD_HSMP) += amd_hsmp.o obj-$(CONFIG_AMD_PMF) += pmf/ +obj-$(CONFIG_AMD_WBRF) += wbrf.o diff --git a/drivers/platform/x86/amd/wbrf.c b/drivers/platform/x86/amd/wbrf.c new file mode 100644 index 00000000000000..dd197b3aebe06a --- /dev/null +++ b/drivers/platform/x86/amd/wbrf.c @@ -0,0 +1,317 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Wifi Frequency Band Manage Interface + * Copyright (C) 2023 Advanced Micro Devices + */ + +#include +#include + +/* + * Functions bit vector for WBRF method + * + * Bit 0: WBRF supported. + * Bit 1: Function 1 (Add / Remove frequency) is supported. + * Bit 2: Function 2 (Get frequency list) is supported. + */ +#define WBRF_ENABLED 0x0 +#define WBRF_RECORD 0x1 +#define WBRF_RETRIEVE 0x2 + +#define WBRF_REVISION 0x1 + +/* + * The data structure used for WBRF_RETRIEVE is not naturally aligned. + * And unfortunately the design has been settled down. + */ +struct amd_wbrf_ranges_out { + u32 num_of_ranges; + struct freq_band_range band_list[MAX_NUM_OF_WBRF_RANGES]; +} __packed; + +static const guid_t wifi_acpi_dsm_guid = + GUID_INIT(0x7b7656cf, 0xdc3d, 0x4c1c, + 0x83, 0xe9, 0x66, 0xe7, 0x21, 0xde, 0x30, 0x70); + +/* + * Used to notify consumer (amdgpu driver currently) about + * the wifi frequency is change. + */ +static BLOCKING_NOTIFIER_HEAD(wbrf_chain_head); + +static int wbrf_record(struct acpi_device *adev, uint8_t action, struct wbrf_ranges_in_out *in) +{ + union acpi_object argv4; + union acpi_object *tmp; + union acpi_object *obj; + u32 num_of_ranges = 0; + u32 num_of_elements; + u32 arg_idx = 0; + int ret; + u32 i; + + if (!in) + return -EINVAL; + + for (i = 0; i < ARRAY_SIZE(in->band_list); i++) { + if (in->band_list[i].start && in->band_list[i].end) + num_of_ranges++; + } + + /* + * The num_of_ranges value in the "in" object supplied by + * the caller is required to be equal to the number of + * entries in the band_list array in there. + */ + if (num_of_ranges != in->num_of_ranges) + return -EINVAL; + + /* + * Every input frequency band comes with two end points(start/end) + * and each is accounted as an element. Meanwhile the range count + * and action type are accounted as an element each. + * So, the total element count = 2 * num_of_ranges + 1 + 1. + */ + num_of_elements = 2 * num_of_ranges + 2; + + tmp = kcalloc(num_of_elements, sizeof(*tmp), GFP_KERNEL); + if (!tmp) + return -ENOMEM; + + argv4.package.type = ACPI_TYPE_PACKAGE; + argv4.package.count = num_of_elements; + argv4.package.elements = tmp; + + /* save the number of ranges*/ + tmp[0].integer.type = ACPI_TYPE_INTEGER; + tmp[0].integer.value = num_of_ranges; + + /* save the action(WBRF_RECORD_ADD/REMOVE/RETRIEVE) */ + tmp[1].integer.type = ACPI_TYPE_INTEGER; + tmp[1].integer.value = action; + + arg_idx = 2; + for (i = 0; i < ARRAY_SIZE(in->band_list); i++) { + if (!in->band_list[i].start || !in->band_list[i].end) + continue; + + tmp[arg_idx].integer.type = ACPI_TYPE_INTEGER; + tmp[arg_idx++].integer.value = in->band_list[i].start; + tmp[arg_idx].integer.type = ACPI_TYPE_INTEGER; + tmp[arg_idx++].integer.value = in->band_list[i].end; + } + + obj = acpi_evaluate_dsm(adev->handle, &wifi_acpi_dsm_guid, + WBRF_REVISION, WBRF_RECORD, &argv4); + + if (!obj) + return -EINVAL; + + if (obj->type != ACPI_TYPE_INTEGER) { + ret = -EINVAL; + goto out; + } + + ret = obj->integer.value; + if (ret) + ret = -EINVAL; + +out: + ACPI_FREE(obj); + kfree(tmp); + + return ret; +} + +/** + * acpi_amd_wbrf_add_remove - add or remove the frequency band the device is using + * + * @dev: device pointer + * @action: remove or add the frequency band into bios + * @in: input structure containing the frequency band the device is using + * + * Broadcast to other consumers the frequency band the device starts + * to use. Underneath the surface the information is cached into an + * internal buffer first. Then a notification is sent to all those + * registered consumers. So then they can retrieve that buffer to + * know the latest active frequency bands. Consumers that haven't + * yet been registered can retrieve the information from the cache + * when they register. + * + * Return: + * 0 for success add/remove wifi frequency band. + * Returns a negative error code for failure. + */ +int acpi_amd_wbrf_add_remove(struct device *dev, uint8_t action, struct wbrf_ranges_in_out *in) +{ + struct acpi_device *adev; + int ret; + + adev = ACPI_COMPANION(dev); + if (!adev) + return -ENODEV; + + ret = wbrf_record(adev, action, in); + if (ret) + return ret; + + blocking_notifier_call_chain(&wbrf_chain_head, WBRF_CHANGED, NULL); + + return 0; +} +EXPORT_SYMBOL_GPL(acpi_amd_wbrf_add_remove); + +/** + * acpi_amd_wbrf_supported_producer - determine if the WBRF can be enabled + * for the device as a producer + * + * @dev: device pointer + * + * Check if the platform equipped with necessary implementations to + * support WBRF for the device as a producer. + * + * Return: + * true if WBRF is supported, otherwise returns false + */ +bool acpi_amd_wbrf_supported_producer(struct device *dev) +{ + struct acpi_device *adev; + + adev = ACPI_COMPANION(dev); + if (!adev) + return false; + + return acpi_check_dsm(adev->handle, &wifi_acpi_dsm_guid, + WBRF_REVISION, BIT(WBRF_RECORD)); +} +EXPORT_SYMBOL_GPL(acpi_amd_wbrf_supported_producer); + +/** + * acpi_amd_wbrf_supported_consumer - determine if the WBRF can be enabled + * for the device as a consumer + * + * @dev: device pointer + * + * Determine if the platform equipped with necessary implementations to + * support WBRF for the device as a consumer. + * + * Return: + * true if WBRF is supported, otherwise returns false. + */ +bool acpi_amd_wbrf_supported_consumer(struct device *dev) +{ + struct acpi_device *adev; + + adev = ACPI_COMPANION(dev); + if (!adev) + return false; + + return acpi_check_dsm(adev->handle, &wifi_acpi_dsm_guid, + WBRF_REVISION, BIT(WBRF_RETRIEVE)); +} +EXPORT_SYMBOL_GPL(acpi_amd_wbrf_supported_consumer); + +/** + * amd_wbrf_retrieve_freq_band - retrieve current active frequency bands + * + * @dev: device pointer + * @out: output structure containing all the active frequency bands + * + * Retrieve the current active frequency bands which were broadcasted + * by other producers. The consumer who calls this API should take + * proper actions if any of the frequency band may cause RFI with its + * own frequency band used. + * + * Return: + * 0 for getting wifi freq band successfully. + * Returns a negative error code for failure. + */ +int amd_wbrf_retrieve_freq_band(struct device *dev, struct wbrf_ranges_in_out *out) +{ + struct amd_wbrf_ranges_out acpi_out = {0}; + struct acpi_device *adev; + union acpi_object *obj; + union acpi_object param; + int ret = 0; + + adev = ACPI_COMPANION(dev); + if (!adev) + return -ENODEV; + + param.type = ACPI_TYPE_STRING; + param.string.length = 0; + param.string.pointer = NULL; + + obj = acpi_evaluate_dsm(adev->handle, &wifi_acpi_dsm_guid, + WBRF_REVISION, WBRF_RETRIEVE, ¶m); + if (!obj) + return -EINVAL; + + /* + * The return buffer is with variable length and the format below: + * number_of_entries(1 DWORD): Number of entries + * start_freq of 1st entry(1 QWORD): Start frequency of the 1st entry + * end_freq of 1st entry(1 QWORD): End frequency of the 1st entry + * ... + * ... + * start_freq of the last entry(1 QWORD) + * end_freq of the last entry(1 QWORD) + * + * Thus the buffer length is determined by the number of entries. + * - For zero entry scenario, the buffer length will be 4 bytes. + * - For one entry scenario, the buffer length will be 20 bytes. + */ + if (obj->buffer.length > sizeof(acpi_out) || obj->buffer.length < 4) { + dev_err(dev, "Wrong sized WBRT information"); + ret = -EINVAL; + goto out; + } + memcpy(&acpi_out, obj->buffer.pointer, obj->buffer.length); + + out->num_of_ranges = acpi_out.num_of_ranges; + memcpy(out->band_list, acpi_out.band_list, sizeof(acpi_out.band_list)); + +out: + ACPI_FREE(obj); + return ret; +} +EXPORT_SYMBOL_GPL(amd_wbrf_retrieve_freq_band); + +/** + * amd_wbrf_register_notifier - register for notifications of frequency + * band update + * + * @nb: driver notifier block + * + * The consumer should register itself via this API so that it can get + * notified on the frequency band updates from other producers. + * + * Return: + * 0 for registering a consumer driver successfully. + * Returns a negative error code for failure. + */ +int amd_wbrf_register_notifier(struct notifier_block *nb) +{ + return blocking_notifier_chain_register(&wbrf_chain_head, nb); +} +EXPORT_SYMBOL_GPL(amd_wbrf_register_notifier); + +/** + * amd_wbrf_unregister_notifier - unregister for notifications of + * frequency band update + * + * @nb: driver notifier block + * + * The consumer should call this API when it is longer interested with + * the frequency band updates from other producers. Usually, this should + * be performed during driver cleanup. + * + * Return: + * 0 for unregistering a consumer driver. + * Returns a negative error code for failure. + */ +int amd_wbrf_unregister_notifier(struct notifier_block *nb) +{ + return blocking_notifier_chain_unregister(&wbrf_chain_head, nb); +} +EXPORT_SYMBOL_GPL(amd_wbrf_unregister_notifier); diff --git a/drivers/vhost/vsock.c b/drivers/vhost/vsock.c index f75731396b7eff..ec20ecff85c7f4 100644 --- a/drivers/vhost/vsock.c +++ b/drivers/vhost/vsock.c @@ -449,6 +449,7 @@ static struct virtio_transport vhost_transport = { .notify_send_pre_enqueue = virtio_transport_notify_send_pre_enqueue, .notify_send_post_enqueue = virtio_transport_notify_send_post_enqueue, .notify_buffer_size = virtio_transport_notify_buffer_size, + .notify_set_rcvlowat = virtio_transport_notify_set_rcvlowat, .read_skb = virtio_transport_read_skb, }, diff --git a/fs/verity/fsverity_private.h b/fs/verity/fsverity_private.h index d071a6e32581e3..a6a6b274924149 100644 --- a/fs/verity/fsverity_private.h +++ b/fs/verity/fsverity_private.h @@ -100,6 +100,16 @@ fsverity_msg(const struct inode *inode, const char *level, #define fsverity_err(inode, fmt, ...) \ fsverity_msg((inode), KERN_ERR, fmt, ##__VA_ARGS__) +/* measure.c */ + +#ifdef CONFIG_BPF_SYSCALL +void __init fsverity_init_bpf(void); +#else +static inline void fsverity_init_bpf(void) +{ +} +#endif + /* open.c */ int fsverity_init_merkle_tree_params(struct merkle_tree_params *params, diff --git a/fs/verity/init.c b/fs/verity/init.c index a29f062f6047b8..1e207c0f71de28 100644 --- a/fs/verity/init.c +++ b/fs/verity/init.c @@ -69,6 +69,7 @@ static int __init fsverity_init(void) fsverity_init_workqueue(); fsverity_init_sysctl(); fsverity_init_signature(); + fsverity_init_bpf(); return 0; } late_initcall(fsverity_init) diff --git a/fs/verity/measure.c b/fs/verity/measure.c index eec5956141da25..bf7a5f4cccaf04 100644 --- a/fs/verity/measure.c +++ b/fs/verity/measure.c @@ -7,6 +7,8 @@ #include "fsverity_private.h" +#include +#include #include /** @@ -100,3 +102,85 @@ int fsverity_get_digest(struct inode *inode, return hash_alg->digest_size; } EXPORT_SYMBOL_GPL(fsverity_get_digest); + +#ifdef CONFIG_BPF_SYSCALL + +/* bpf kfuncs */ +__bpf_kfunc_start_defs(); + +/** + * bpf_get_fsverity_digest: read fsverity digest of file + * @file: file to get digest from + * @digest_ptr: (out) dynptr for struct fsverity_digest + * + * Read fsverity_digest of *file* into *digest_ptr*. + * + * Return: 0 on success, a negative value on error. + */ +__bpf_kfunc int bpf_get_fsverity_digest(struct file *file, struct bpf_dynptr_kern *digest_ptr) +{ + const struct inode *inode = file_inode(file); + u32 dynptr_sz = __bpf_dynptr_size(digest_ptr); + struct fsverity_digest *arg; + const struct fsverity_info *vi; + const struct fsverity_hash_alg *hash_alg; + int out_digest_sz; + + if (dynptr_sz < sizeof(struct fsverity_digest)) + return -EINVAL; + + arg = __bpf_dynptr_data_rw(digest_ptr, dynptr_sz); + if (!arg) + return -EINVAL; + + if (!IS_ALIGNED((uintptr_t)arg, __alignof__(*arg))) + return -EINVAL; + + vi = fsverity_get_info(inode); + if (!vi) + return -ENODATA; /* not a verity file */ + + hash_alg = vi->tree_params.hash_alg; + + arg->digest_algorithm = hash_alg - fsverity_hash_algs; + arg->digest_size = hash_alg->digest_size; + + out_digest_sz = dynptr_sz - sizeof(struct fsverity_digest); + + /* copy digest */ + memcpy(arg->digest, vi->file_digest, min_t(int, hash_alg->digest_size, out_digest_sz)); + + /* fill the extra buffer with zeros */ + if (out_digest_sz > hash_alg->digest_size) + memset(arg->digest + arg->digest_size, 0, out_digest_sz - hash_alg->digest_size); + + return 0; +} + +__bpf_kfunc_end_defs(); + +BTF_SET8_START(fsverity_set_ids) +BTF_ID_FLAGS(func, bpf_get_fsverity_digest, KF_TRUSTED_ARGS) +BTF_SET8_END(fsverity_set_ids) + +static int bpf_get_fsverity_digest_filter(const struct bpf_prog *prog, u32 kfunc_id) +{ + if (!btf_id_set8_contains(&fsverity_set_ids, kfunc_id)) + return 0; + + /* Only allow to attach from LSM hooks, to avoid recursion */ + return prog->type != BPF_PROG_TYPE_LSM ? -EACCES : 0; +} + +static const struct btf_kfunc_id_set bpf_fsverity_set = { + .owner = THIS_MODULE, + .set = &fsverity_set_ids, + .filter = bpf_get_fsverity_digest_filter, +}; + +void __init fsverity_init_bpf(void) +{ + register_btf_kfunc_id_set(BPF_PROG_TYPE_LSM, &bpf_fsverity_set); +} + +#endif /* CONFIG_BPF_SYSCALL */ diff --git a/include/asm-generic/Kbuild b/include/asm-generic/Kbuild index def242528b1db4..d436bee4d129dd 100644 --- a/include/asm-generic/Kbuild +++ b/include/asm-generic/Kbuild @@ -11,6 +11,7 @@ mandatory-y += bitops.h mandatory-y += bug.h mandatory-y += bugs.h mandatory-y += cacheflush.h +mandatory-y += cfi.h mandatory-y += checksum.h mandatory-y += compat.h mandatory-y += current.h diff --git a/include/asm-generic/cfi.h b/include/asm-generic/cfi.h new file mode 100644 index 00000000000000..41fac3537bf94d --- /dev/null +++ b/include/asm-generic/cfi.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_GENERIC_CFI_H +#define __ASM_GENERIC_CFI_H + +#endif /* __ASM_GENERIC_CFI_H */ diff --git a/include/linux/acpi_amd_wbrf.h b/include/linux/acpi_amd_wbrf.h new file mode 100644 index 00000000000000..898f31d536d4ac --- /dev/null +++ b/include/linux/acpi_amd_wbrf.h @@ -0,0 +1,91 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Wifi Band Exclusion Interface (AMD ACPI Implementation) + * Copyright (C) 2023 Advanced Micro Devices + */ + +#ifndef _ACPI_AMD_WBRF_H +#define _ACPI_AMD_WBRF_H + +#include +#include + +/* The maximum number of frequency band ranges */ +#define MAX_NUM_OF_WBRF_RANGES 11 + +/* Record actions */ +#define WBRF_RECORD_ADD 0x0 +#define WBRF_RECORD_REMOVE 0x1 + +/** + * struct freq_band_range - Wifi frequency band range definition + * @start: start frequency point (in Hz) + * @end: end frequency point (in Hz) + */ +struct freq_band_range { + u64 start; + u64 end; +}; + +/** + * struct wbrf_ranges_in_out - wbrf ranges info + * @num_of_ranges: total number of band ranges in this struct + * @band_list: array of Wifi band ranges + */ +struct wbrf_ranges_in_out { + u64 num_of_ranges; + struct freq_band_range band_list[MAX_NUM_OF_WBRF_RANGES]; +}; + +/** + * enum wbrf_notifier_actions - wbrf notifier actions index + * @WBRF_CHANGED: there was some frequency band updates. The consumers + * should retrieve the latest active frequency bands. + */ +enum wbrf_notifier_actions { + WBRF_CHANGED, +}; + +#if IS_ENABLED(CONFIG_AMD_WBRF) +bool acpi_amd_wbrf_supported_producer(struct device *dev); +int acpi_amd_wbrf_add_remove(struct device *dev, uint8_t action, struct wbrf_ranges_in_out *in); +bool acpi_amd_wbrf_supported_consumer(struct device *dev); +int amd_wbrf_retrieve_freq_band(struct device *dev, struct wbrf_ranges_in_out *out); +int amd_wbrf_register_notifier(struct notifier_block *nb); +int amd_wbrf_unregister_notifier(struct notifier_block *nb); +#else +static inline +bool acpi_amd_wbrf_supported_consumer(struct device *dev) +{ + return false; +} + +static inline +int acpi_amd_wbrf_add_remove(struct device *dev, uint8_t action, struct wbrf_ranges_in_out *in) +{ + return -ENODEV; +} + +static inline +bool acpi_amd_wbrf_supported_producer(struct device *dev) +{ + return false; +} +static inline +int amd_wbrf_retrieve_freq_band(struct device *dev, struct wbrf_ranges_in_out *out) +{ + return -ENODEV; +} +static inline +int amd_wbrf_register_notifier(struct notifier_block *nb) +{ + return -ENODEV; +} +static inline +int amd_wbrf_unregister_notifier(struct notifier_block *nb) +{ + return -ENODEV; +} +#endif /* CONFIG_AMD_WBRF */ + +#endif /* _ACPI_AMD_WBRF_H */ diff --git a/include/linux/bpf.h b/include/linux/bpf.h index d5d8e2083610be..2f54cc0436c4d2 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -29,6 +29,7 @@ #include #include #include +#include struct bpf_verifier_env; struct bpf_verifier_log; @@ -51,6 +52,10 @@ struct module; struct bpf_func_state; struct ftrace_ops; struct cgroup; +struct bpf_token; +struct user_namespace; +struct super_block; +struct inode; extern struct idr btf_idr; extern spinlock_t btf_idr_lock; @@ -106,7 +111,11 @@ struct bpf_map_ops { /* funcs called by prog_array and perf_event_array map */ void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file, int fd); - void (*map_fd_put_ptr)(void *ptr); + /* If need_defer is true, the implementation should guarantee that + * the to-be-put element is still alive before the bpf program, which + * may manipulate it, exists. + */ + void (*map_fd_put_ptr)(struct bpf_map *map, void *ptr, bool need_defer); int (*map_gen_lookup)(struct bpf_map *map, struct bpf_insn *insn_buf); u32 (*map_fd_sys_lookup_elem)(void *ptr); void (*map_seq_show_elem)(struct bpf_map *map, void *key, @@ -272,7 +281,11 @@ struct bpf_map { */ atomic64_t refcnt ____cacheline_aligned; atomic64_t usercnt; - struct work_struct work; + /* rcu is used before freeing and work is only used during freeing */ + union { + struct work_struct work; + struct rcu_head rcu; + }; struct mutex freeze_mutex; atomic64_t writecnt; /* 'Ownership' of program-containing map is claimed by the first program @@ -288,6 +301,9 @@ struct bpf_map { } owner; bool bypass_spec_v1; bool frozen; /* write-once; write-protected by freeze_mutex */ + bool free_after_mult_rcu_gp; + bool free_after_rcu_gp; + atomic64_t sleepable_refcnt; s64 __percpu *elem_count; }; @@ -1044,6 +1060,17 @@ struct btf_func_model { */ #define BPF_TRAMP_F_TAIL_CALL_CTX BIT(7) +/* + * Indicate the trampoline should be suitable to receive indirect calls; + * without this indirectly calling the generated code can result in #UD/#CP, + * depending on the CFI options. + * + * Used by bpf_struct_ops. + * + * Incompatible with FENTRY usage, overloads @func_addr argument. + */ +#define BPF_TRAMP_F_INDIRECT BIT(8) + /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50 * bytes on x86. */ @@ -1083,10 +1110,17 @@ struct bpf_tramp_run_ctx; * fexit = a set of program to run after original function */ struct bpf_tramp_image; -int arch_prepare_bpf_trampoline(struct bpf_tramp_image *tr, void *image, void *image_end, +int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, const struct btf_func_model *m, u32 flags, struct bpf_tramp_links *tlinks, - void *orig_call); + void *func_addr); +void *arch_alloc_bpf_trampoline(unsigned int size); +void arch_free_bpf_trampoline(void *image, unsigned int size); +void arch_protect_bpf_trampoline(void *image, unsigned int size); +void arch_unprotect_bpf_trampoline(void *image, unsigned int size); +int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, + struct bpf_tramp_links *tlinks, void *func_addr); + u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx); void notrace __bpf_prog_exit_sleepable_recur(struct bpf_prog *prog, u64 start, @@ -1119,6 +1153,7 @@ enum bpf_tramp_prog_type { struct bpf_tramp_image { void *image; + int size; struct bpf_ksym ksym; struct percpu_ref pcref; void *ip_after_call; @@ -1188,7 +1223,11 @@ struct bpf_dispatcher { #endif }; -static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func( +#ifndef __bpfcall +#define __bpfcall __nocfi +#endif + +static __always_inline __bpfcall unsigned int bpf_dispatcher_nop_func( const void *ctx, const struct bpf_insn *insnsi, bpf_func_t bpf_func) @@ -1280,7 +1319,7 @@ int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_func #define DEFINE_BPF_DISPATCHER(name) \ __BPF_DISPATCHER_SC(name); \ - noinline __nocfi unsigned int bpf_dispatcher_##name##_func( \ + noinline __bpfcall unsigned int bpf_dispatcher_##name##_func( \ const void *ctx, \ const struct bpf_insn *insnsi, \ bpf_func_t bpf_func) \ @@ -1303,7 +1342,7 @@ int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_func void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, struct bpf_prog *to); /* Called only from JIT-enabled code, so there's no need for stubs. */ -void bpf_image_ksym_add(void *data, struct bpf_ksym *ksym); +void bpf_image_ksym_add(void *data, unsigned int size, struct bpf_ksym *ksym); void bpf_image_ksym_del(struct bpf_ksym *ksym); void bpf_ksym_add(struct bpf_ksym *ksym); void bpf_ksym_del(struct bpf_ksym *ksym); @@ -1430,6 +1469,9 @@ struct bpf_prog_aux { struct bpf_kfunc_desc_tab *kfunc_tab; struct bpf_kfunc_btf_tab *kfunc_btf_tab; u32 size_poke_tab; +#ifdef CONFIG_FINEIBT + struct bpf_ksym ksym_prefix; +#endif struct bpf_ksym ksym; const struct bpf_prog_ops *ops; struct bpf_map **used_maps; @@ -1442,10 +1484,11 @@ struct bpf_prog_aux { int cgroup_atype; /* enum cgroup_bpf_attach_type */ struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]; char name[BPF_OBJ_NAME_LEN]; - unsigned int (*bpf_exception_cb)(u64 cookie, u64 sp, u64 bp); + u64 (*bpf_exception_cb)(u64 cookie, u64 sp, u64 bp, u64, u64); #ifdef CONFIG_SECURITY void *security; #endif + struct bpf_token *token; struct bpf_prog_offload *offload; struct btf *btf; struct bpf_func_info *func_info; @@ -1570,6 +1613,31 @@ struct bpf_link_primer { u32 id; }; +struct bpf_mount_opts { + kuid_t uid; + kgid_t gid; + umode_t mode; + + /* BPF token-related delegation options */ + u64 delegate_cmds; + u64 delegate_maps; + u64 delegate_progs; + u64 delegate_attachs; +}; + +struct bpf_token { + struct work_struct work; + atomic64_t refcnt; + struct user_namespace *userns; + u64 allowed_cmds; + u64 allowed_maps; + u64 allowed_progs; + u64 allowed_attachs; +#ifdef CONFIG_SECURITY + void *security; +#endif +}; + struct bpf_struct_ops_value; struct btf_member; @@ -1640,6 +1708,7 @@ struct bpf_struct_ops { struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS]; u32 type_id; u32 value_id; + void *cfi_stubs; }; #if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL) @@ -1653,6 +1722,7 @@ int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key, int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks, struct bpf_tramp_link *link, const struct btf_func_model *model, + void *stub_func, void *image, void *image_end); static inline bool bpf_try_module_get(const void *data, struct module *owner) { @@ -2027,6 +2097,7 @@ static inline void bpf_enable_instrumentation(void) migrate_enable(); } +extern const struct super_operations bpf_super_ops; extern const struct file_operations bpf_map_fops; extern const struct file_operations bpf_prog_fops; extern const struct file_operations bpf_iter_fops; @@ -2161,24 +2232,26 @@ static inline void bpf_map_dec_elem_count(struct bpf_map *map) extern int sysctl_unprivileged_bpf_disabled; -static inline bool bpf_allow_ptr_leaks(void) +bool bpf_token_capable(const struct bpf_token *token, int cap); + +static inline bool bpf_allow_ptr_leaks(const struct bpf_token *token) { - return perfmon_capable(); + return bpf_token_capable(token, CAP_PERFMON); } -static inline bool bpf_allow_uninit_stack(void) +static inline bool bpf_allow_uninit_stack(const struct bpf_token *token) { - return perfmon_capable(); + return bpf_token_capable(token, CAP_PERFMON); } -static inline bool bpf_bypass_spec_v1(void) +static inline bool bpf_bypass_spec_v1(const struct bpf_token *token) { - return cpu_mitigations_off() || perfmon_capable(); + return cpu_mitigations_off() || bpf_token_capable(token, CAP_PERFMON); } -static inline bool bpf_bypass_spec_v4(void) +static inline bool bpf_bypass_spec_v4(const struct bpf_token *token) { - return cpu_mitigations_off() || perfmon_capable(); + return cpu_mitigations_off() || bpf_token_capable(token, CAP_PERFMON); } int bpf_map_new_fd(struct bpf_map *map, int flags); @@ -2195,8 +2268,21 @@ int bpf_link_new_fd(struct bpf_link *link); struct bpf_link *bpf_link_get_from_fd(u32 ufd); struct bpf_link *bpf_link_get_curr_or_next(u32 *id); +void bpf_token_inc(struct bpf_token *token); +void bpf_token_put(struct bpf_token *token); +int bpf_token_create(union bpf_attr *attr); +struct bpf_token *bpf_token_get_from_fd(u32 ufd); + +bool bpf_token_allow_cmd(const struct bpf_token *token, enum bpf_cmd cmd); +bool bpf_token_allow_map_type(const struct bpf_token *token, enum bpf_map_type type); +bool bpf_token_allow_prog_type(const struct bpf_token *token, + enum bpf_prog_type prog_type, + enum bpf_attach_type attach_type); + int bpf_obj_pin_user(u32 ufd, int path_fd, const char __user *pathname); int bpf_obj_get_user(int path_fd, const char __user *pathname, int flags); +struct inode *bpf_get_inode(struct super_block *sb, const struct inode *dir, + umode_t mode); #define BPF_ITER_FUNC_PREFIX "bpf_iter_" #define DEFINE_BPF_ITER_FUNC(target, args...) \ @@ -2431,7 +2517,7 @@ int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog, int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog, struct bpf_reg_state *regs); int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, - struct bpf_reg_state *reg, bool is_ex_cb); + struct bpf_reg_state *reg, u32 *nargs); int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *prog, struct btf *btf, const struct btf_type *t); const char *btf_find_decl_tag_value(const struct btf *btf, const struct btf_type *pt, @@ -2440,7 +2526,8 @@ const char *btf_find_decl_tag_value(const struct btf *btf, const struct btf_type struct bpf_prog *bpf_prog_by_id(u32 id); struct bpf_link *bpf_link_by_id(u32 id); -const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id); +const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id, + const struct bpf_prog *prog); void bpf_task_storage_free(struct task_struct *task); void bpf_cgrp_storage_free(struct cgroup *cgroup); bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog); @@ -2559,6 +2646,24 @@ static inline int bpf_obj_get_user(const char __user *pathname, int flags) return -EOPNOTSUPP; } +static inline bool bpf_token_capable(const struct bpf_token *token, int cap) +{ + return capable(cap) || (cap != CAP_SYS_ADMIN && capable(CAP_SYS_ADMIN)); +} + +static inline void bpf_token_inc(struct bpf_token *token) +{ +} + +static inline void bpf_token_put(struct bpf_token *token) +{ +} + +static inline struct bpf_token *bpf_token_get_from_fd(u32 ufd) +{ + return ERR_PTR(-EOPNOTSUPP); +} + static inline void __dev_flush(void) { } @@ -2682,7 +2787,7 @@ static inline int btf_struct_access(struct bpf_verifier_log *log, } static inline const struct bpf_func_proto * -bpf_base_func_proto(enum bpf_func_id func_id) +bpf_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { return NULL; } diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index d99a636d36a7cd..c2819a6579a520 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -275,6 +275,11 @@ struct bpf_reference_state { int callback_ref; }; +struct bpf_retval_range { + s32 minval; + s32 maxval; +}; + /* state of the program: * type of all registers and stack info */ @@ -297,8 +302,8 @@ struct bpf_func_state { * void foo(void) { bpf_timer_set_callback(,foo); } */ u32 async_entry_cnt; + struct bpf_retval_range callback_ret_range; bool in_callback_fn; - struct tnum callback_ret_range; bool in_async_callback_fn; bool in_exception_callback_fn; /* For callback calling functions that limit number of possible @@ -316,16 +321,48 @@ struct bpf_func_state { /* The following fields should be last. See copy_func_state() */ int acquired_refs; struct bpf_reference_state *refs; - int allocated_stack; + /* The state of the stack. Each element of the array describes BPF_REG_SIZE + * (i.e. 8) bytes worth of stack memory. + * stack[0] represents bytes [*(r10-8)..*(r10-1)] + * stack[1] represents bytes [*(r10-16)..*(r10-9)] + * ... + * stack[allocated_stack/8 - 1] represents [*(r10-allocated_stack)..*(r10-allocated_stack+7)] + */ struct bpf_stack_state *stack; + /* Size of the current stack, in bytes. The stack state is tracked below, in + * `stack`. allocated_stack is always a multiple of BPF_REG_SIZE. + */ + int allocated_stack; +}; + +#define MAX_CALL_FRAMES 8 + +/* instruction history flags, used in bpf_jmp_history_entry.flags field */ +enum { + /* instruction references stack slot through PTR_TO_STACK register; + * we also store stack's frame number in lower 3 bits (MAX_CALL_FRAMES is 8) + * and accessed stack slot's index in next 6 bits (MAX_BPF_STACK is 512, + * 8 bytes per slot, so slot index (spi) is [0, 63]) + */ + INSN_F_FRAMENO_MASK = 0x7, /* 3 bits */ + + INSN_F_SPI_MASK = 0x3f, /* 6 bits */ + INSN_F_SPI_SHIFT = 3, /* shifted 3 bits to the left */ + + INSN_F_STACK_ACCESS = BIT(9), /* we need 10 bits total */ }; -struct bpf_idx_pair { - u32 prev_idx; +static_assert(INSN_F_FRAMENO_MASK + 1 >= MAX_CALL_FRAMES); +static_assert(INSN_F_SPI_MASK + 1 >= MAX_BPF_STACK / 8); + +struct bpf_jmp_history_entry { u32 idx; + /* insn idx can't be bigger than 1 million */ + u32 prev_idx : 22; + /* special flags, e.g., whether insn is doing register stack spill/load */ + u32 flags : 10; }; -#define MAX_CALL_FRAMES 8 /* Maximum number of register states that can exist at once */ #define BPF_ID_MAP_SIZE ((MAX_BPF_REG + MAX_BPF_STACK / BPF_REG_SIZE) * MAX_CALL_FRAMES) struct bpf_verifier_state { @@ -408,7 +445,7 @@ struct bpf_verifier_state { * For most states jmp_history_cnt is [0-3]. * For loops can go up to ~40. */ - struct bpf_idx_pair *jmp_history; + struct bpf_jmp_history_entry *jmp_history; u32 jmp_history_cnt; u32 dfs_depth; u32 callback_unroll_depth; @@ -574,12 +611,12 @@ struct bpf_subprog_info { u32 start; /* insn idx of function entry point */ u32 linfo_idx; /* The idx to the main_prog->aux->linfo */ u16 stack_depth; /* max. stack depth used by this function */ - bool has_tail_call; - bool tail_call_reachable; - bool has_ld_abs; - bool is_cb; - bool is_async_cb; - bool is_exception_cb; + bool has_tail_call: 1; + bool tail_call_reachable: 1; + bool has_ld_abs: 1; + bool is_cb: 1; + bool is_async_cb: 1; + bool is_exception_cb: 1; }; struct bpf_verifier_env; @@ -631,6 +668,10 @@ struct bpf_verifier_env { int exception_callback_subprog; bool explore_alu_limits; bool allow_ptr_leaks; + /* Allow access to uninitialized stack memory. Writes with fixed offset are + * always allowed, so this refers to reads (with fixed or variable offset), + * to writes with variable offset and to indirect (helper) accesses. + */ bool allow_uninit_stack; bool bpf_capable; bool bypass_spec_v1; @@ -651,6 +692,7 @@ struct bpf_verifier_env { int cur_stack; } cfg; struct backtrack_state bt; + struct bpf_jmp_history_entry *cur_hist_ent; u32 pass_cnt; /* number of times do_check() was called */ u32 subprog_cnt; /* number of instructions analyzed by the verifier */ diff --git a/include/linux/cfi.h b/include/linux/cfi.h index 3552ec82b72561..f0df518e11dd15 100644 --- a/include/linux/cfi.h +++ b/include/linux/cfi.h @@ -9,6 +9,14 @@ #include #include +#include + +#ifndef cfi_get_offset +static inline int cfi_get_offset(void) +{ + return 0; +} +#endif #ifdef CONFIG_CFI_CLANG enum bug_trap_type report_cfi_failure(struct pt_regs *regs, unsigned long addr, @@ -38,4 +46,8 @@ static inline void module_cfi_finalize(const Elf_Ehdr *hdr, #endif /* CONFIG_ARCH_USES_CFI_TRAPS */ #endif /* CONFIG_MODULES */ +#ifndef CFI_NOSEAL +#define CFI_NOSEAL(x) +#endif + #endif /* _LINUX_CFI_H */ diff --git a/include/linux/filter.h b/include/linux/filter.h index a4953fafc8cb8c..12d907f17d3642 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1067,7 +1067,7 @@ struct bpf_binary_header * bpf_jit_binary_pack_hdr(const struct bpf_prog *fp); void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns); -void bpf_prog_pack_free(struct bpf_binary_header *hdr); +void bpf_prog_pack_free(void *ptr, u32 size); static inline bool bpf_prog_kallsyms_verify_off(const struct bpf_prog *fp) { @@ -1139,7 +1139,7 @@ static inline bool bpf_jit_blinding_enabled(struct bpf_prog *prog) return false; if (!bpf_jit_harden) return false; - if (bpf_jit_harden == 1 && bpf_capable()) + if (bpf_jit_harden == 1 && bpf_token_capable(prog->aux->token, CAP_BPF)) return false; return true; diff --git a/include/linux/if_vlan.h b/include/linux/if_vlan.h index 3028af87716e29..c1645c86eed969 100644 --- a/include/linux/if_vlan.h +++ b/include/linux/if_vlan.h @@ -540,7 +540,7 @@ static inline int __vlan_get_tag(const struct sk_buff *skb, u16 *vlan_tci) struct vlan_ethhdr *veth = skb_vlan_eth_hdr(skb); if (!eth_type_vlan(veth->h_vlan_proto)) - return -EINVAL; + return -ENODATA; *vlan_tci = ntohs(veth->h_vlan_TCI); return 0; @@ -561,7 +561,7 @@ static inline int __vlan_hwaccel_get_tag(const struct sk_buff *skb, return 0; } else { *vlan_tci = 0; - return -EINVAL; + return -ENODATA; } } diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index ff217a5ce55214..3fdd00b452acab 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -398,10 +398,17 @@ LSM_HOOK(void, LSM_RET_VOID, audit_rule_free, void *lsmrule) LSM_HOOK(int, 0, bpf, int cmd, union bpf_attr *attr, unsigned int size) LSM_HOOK(int, 0, bpf_map, struct bpf_map *map, fmode_t fmode) LSM_HOOK(int, 0, bpf_prog, struct bpf_prog *prog) -LSM_HOOK(int, 0, bpf_map_alloc_security, struct bpf_map *map) -LSM_HOOK(void, LSM_RET_VOID, bpf_map_free_security, struct bpf_map *map) -LSM_HOOK(int, 0, bpf_prog_alloc_security, struct bpf_prog_aux *aux) -LSM_HOOK(void, LSM_RET_VOID, bpf_prog_free_security, struct bpf_prog_aux *aux) +LSM_HOOK(int, 0, bpf_map_create, struct bpf_map *map, union bpf_attr *attr, + struct bpf_token *token) +LSM_HOOK(void, LSM_RET_VOID, bpf_map_free, struct bpf_map *map) +LSM_HOOK(int, 0, bpf_prog_load, struct bpf_prog *prog, union bpf_attr *attr, + struct bpf_token *token) +LSM_HOOK(void, LSM_RET_VOID, bpf_prog_free, struct bpf_prog *prog) +LSM_HOOK(int, 0, bpf_token_create, struct bpf_token *token, union bpf_attr *attr, + struct path *path) +LSM_HOOK(void, LSM_RET_VOID, bpf_token_free, struct bpf_token *token) +LSM_HOOK(int, 0, bpf_token_cmd, const struct bpf_token *token, enum bpf_cmd cmd) +LSM_HOOK(int, 0, bpf_token_capable, const struct bpf_token *token, int cap) #endif /* CONFIG_BPF_SYSCALL */ LSM_HOOK(int, 0, locked_down, enum lockdown_reason what) diff --git a/include/linux/mlx5/device.h b/include/linux/mlx5/device.h index 820bca965fb6f2..01275c6e84688f 100644 --- a/include/linux/mlx5/device.h +++ b/include/linux/mlx5/device.h @@ -918,7 +918,7 @@ static inline u8 get_cqe_tls_offload(struct mlx5_cqe64 *cqe) return (cqe->tls_outer_l3_tunneled >> 3) & 0x3; } -static inline bool cqe_has_vlan(struct mlx5_cqe64 *cqe) +static inline bool cqe_has_vlan(const struct mlx5_cqe64 *cqe) { return cqe->l4_l3_hdr_type & 0x1; } diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index d2b8d4a74a3086..7ee5b79ff3d604 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -150,6 +150,7 @@ enum { MLX5_REG_MTPPSE = 0x9054, MLX5_REG_MTUTC = 0x9055, MLX5_REG_MPEGC = 0x9056, + MLX5_REG_MPIR = 0x9059, MLX5_REG_MCQS = 0x9060, MLX5_REG_MCQI = 0x9061, MLX5_REG_MCC = 0x9062, @@ -678,6 +679,8 @@ struct mlx5e_resources { struct mlx5_td td; u32 mkey; struct mlx5_sq_bfreg bfreg; +#define MLX5_MAX_NUM_TC 8 + u32 tisn[MLX5_MAX_PORTS][MLX5_MAX_NUM_TC]; } hw_objs; struct net_device *uplink_netdev; struct mutex uplink_netdev_lock; diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 36633ed21585ba..fee20fc010c2d2 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -435,7 +435,7 @@ struct mlx5_ifc_flow_table_prop_layout_bits { u8 flow_table_modify[0x1]; u8 reformat[0x1]; u8 decap[0x1]; - u8 reserved_at_9[0x1]; + u8 reset_root_to_default[0x1]; u8 pop_vlan[0x1]; u8 push_vlan[0x1]; u8 reserved_at_c[0x1]; @@ -1801,7 +1801,8 @@ struct mlx5_ifc_cmd_hca_cap_bits { u8 disable_local_lb_uc[0x1]; u8 disable_local_lb_mc[0x1]; u8 log_min_hairpin_wq_data_sz[0x5]; - u8 reserved_at_3e8[0x2]; + u8 reserved_at_3e8[0x1]; + u8 silent_mode[0x1]; u8 vhca_state[0x1]; u8 log_max_vlan_list[0x5]; u8 reserved_at_3f0[0x3]; @@ -1818,7 +1819,7 @@ struct mlx5_ifc_cmd_hca_cap_bits { u8 reserved_at_460[0x1]; u8 ats[0x1]; - u8 reserved_at_462[0x1]; + u8 cross_vhca_rqt[0x1]; u8 log_max_uctx[0x5]; u8 reserved_at_468[0x1]; u8 crypto[0x1]; @@ -1943,6 +1944,7 @@ struct mlx5_ifc_cmd_hca_cap_bits { enum { MLX5_CROSS_VHCA_OBJ_TO_OBJ_SUPPORTED_LOCAL_FLOW_TABLE_TO_REMOTE_FLOW_TABLE_MISS = 0x80000, + MLX5_CROSS_VHCA_OBJ_TO_OBJ_SUPPORTED_LOCAL_FLOW_TABLE_ROOT_TO_REMOTE_FLOW_TABLE = (1ULL << 20), }; enum { @@ -1992,7 +1994,11 @@ struct mlx5_ifc_cmd_hca_cap_2_bits { u8 reserved_at_260[0x120]; u8 reserved_at_380[0x10]; u8 ec_vf_vport_base[0x10]; - u8 reserved_at_3a0[0x460]; + + u8 reserved_at_3a0[0x10]; + u8 max_rqt_vhca_id[0x10]; + + u8 reserved_at_3c0[0x440]; }; enum mlx5_ifc_flow_destination_type { @@ -2151,6 +2157,13 @@ struct mlx5_ifc_rq_num_bits { u8 rq_num[0x18]; }; +struct mlx5_ifc_rq_vhca_bits { + u8 reserved_at_0[0x8]; + u8 rq_num[0x18]; + u8 reserved_at_20[0x10]; + u8 rq_vhca_id[0x10]; +}; + struct mlx5_ifc_mac_address_layout_bits { u8 reserved_at_0[0x10]; u8 mac_addr_47_32[0x10]; @@ -3901,7 +3914,10 @@ struct mlx5_ifc_rqtc_bits { u8 reserved_at_e0[0x6a0]; - struct mlx5_ifc_rq_num_bits rq_num[]; + union { + DECLARE_FLEX_ARRAY(struct mlx5_ifc_rq_num_bits, rq_num); + DECLARE_FLEX_ARRAY(struct mlx5_ifc_rq_vhca_bits, rq_vhca); + }; }; enum { @@ -4744,7 +4760,10 @@ struct mlx5_ifc_set_l2_table_entry_in_bits { u8 reserved_at_c0[0x20]; - u8 reserved_at_e0[0x13]; + u8 reserved_at_e0[0x10]; + u8 silent_mode_valid[0x1]; + u8 silent_mode[0x1]; + u8 reserved_at_f2[0x1]; u8 vlan_valid[0x1]; u8 vlan[0xc]; @@ -10089,6 +10108,20 @@ struct mlx5_ifc_mpegc_reg_bits { u8 reserved_at_60[0x100]; }; +struct mlx5_ifc_mpir_reg_bits { + u8 sdm[0x1]; + u8 reserved_at_1[0x1b]; + u8 host_buses[0x4]; + + u8 reserved_at_20[0x20]; + + u8 local_port[0x8]; + u8 reserved_at_28[0x15]; + u8 sd_group[0x3]; + + u8 reserved_at_60[0x20]; +}; + enum { MLX5_MTUTC_FREQ_ADJ_UNITS_PPB = 0x0, MLX5_MTUTC_FREQ_ADJ_UNITS_SCALED_PPM = 0x1, diff --git a/include/linux/phy.h b/include/linux/phy.h index dbb5e13e3e1bf0..e9e85d34758720 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -327,7 +327,8 @@ struct mdio_bus_stats { /** * struct phy_package_shared - Shared information in PHY packages - * @addr: Common PHY address used to combine PHYs in one package + * @base_addr: Base PHY address of PHY package used to combine PHYs + * in one package and for offset calculation of phy_package_read/write * @refcnt: Number of PHYs connected to this shared data * @flags: Initialization of PHY package * @priv_size: Size of the shared private data @priv @@ -338,7 +339,7 @@ struct mdio_bus_stats { * phy_package_leave(). */ struct phy_package_shared { - int addr; + u8 base_addr; refcount_t refcnt; unsigned long flags; size_t priv_size; @@ -1976,10 +1977,10 @@ int phy_ethtool_get_link_ksettings(struct net_device *ndev, int phy_ethtool_set_link_ksettings(struct net_device *ndev, const struct ethtool_link_ksettings *cmd); int phy_ethtool_nway_reset(struct net_device *ndev); -int phy_package_join(struct phy_device *phydev, int addr, size_t priv_size); +int phy_package_join(struct phy_device *phydev, int base_addr, size_t priv_size); void phy_package_leave(struct phy_device *phydev); int devm_phy_package_join(struct device *dev, struct phy_device *phydev, - int addr, size_t priv_size); + int base_addr, size_t priv_size); int __init mdio_bus_init(void); void mdio_bus_exit(void); @@ -2002,48 +2003,83 @@ int __phy_hwtstamp_set(struct phy_device *phydev, struct kernel_hwtstamp_config *config, struct netlink_ext_ack *extack); -static inline int phy_package_read(struct phy_device *phydev, u32 regnum) +static inline int phy_package_address(struct phy_device *phydev, + unsigned int addr_offset) { struct phy_package_shared *shared = phydev->shared; + u8 base_addr = shared->base_addr; - if (!shared) + if (addr_offset >= PHY_MAX_ADDR - base_addr) return -EIO; - return mdiobus_read(phydev->mdio.bus, shared->addr, regnum); + /* we know that addr will be in the range 0..31 and thus the + * implicit cast to a signed int is not a problem. + */ + return base_addr + addr_offset; } -static inline int __phy_package_read(struct phy_device *phydev, u32 regnum) +static inline int phy_package_read(struct phy_device *phydev, + unsigned int addr_offset, u32 regnum) { - struct phy_package_shared *shared = phydev->shared; + int addr = phy_package_address(phydev, addr_offset); - if (!shared) - return -EIO; + if (addr < 0) + return addr; + + return mdiobus_read(phydev->mdio.bus, addr, regnum); +} + +static inline int __phy_package_read(struct phy_device *phydev, + unsigned int addr_offset, u32 regnum) +{ + int addr = phy_package_address(phydev, addr_offset); + + if (addr < 0) + return addr; - return __mdiobus_read(phydev->mdio.bus, shared->addr, regnum); + return __mdiobus_read(phydev->mdio.bus, addr, regnum); } static inline int phy_package_write(struct phy_device *phydev, - u32 regnum, u16 val) + unsigned int addr_offset, u32 regnum, + u16 val) { - struct phy_package_shared *shared = phydev->shared; + int addr = phy_package_address(phydev, addr_offset); - if (!shared) - return -EIO; + if (addr < 0) + return addr; - return mdiobus_write(phydev->mdio.bus, shared->addr, regnum, val); + return mdiobus_write(phydev->mdio.bus, addr, regnum, val); } static inline int __phy_package_write(struct phy_device *phydev, - u32 regnum, u16 val) + unsigned int addr_offset, u32 regnum, + u16 val) { - struct phy_package_shared *shared = phydev->shared; + int addr = phy_package_address(phydev, addr_offset); - if (!shared) - return -EIO; + if (addr < 0) + return addr; - return __mdiobus_write(phydev->mdio.bus, shared->addr, regnum, val); + return __mdiobus_write(phydev->mdio.bus, addr, regnum, val); } +int __phy_package_read_mmd(struct phy_device *phydev, + unsigned int addr_offset, int devad, + u32 regnum); + +int phy_package_read_mmd(struct phy_device *phydev, + unsigned int addr_offset, int devad, + u32 regnum); + +int __phy_package_write_mmd(struct phy_device *phydev, + unsigned int addr_offset, int devad, + u32 regnum, u16 val); + +int phy_package_write_mmd(struct phy_device *phydev, + unsigned int addr_offset, int devad, + u32 regnum, u16 val); + static inline bool __phy_package_set_once(struct phy_device *phydev, unsigned int b) { diff --git a/include/linux/rtnetlink.h b/include/linux/rtnetlink.h index 6a8543b34e2c09..410529fca18b2f 100644 --- a/include/linux/rtnetlink.h +++ b/include/linux/rtnetlink.h @@ -79,6 +79,18 @@ static inline bool lockdep_rtnl_is_held(void) #define rtnl_dereference(p) \ rcu_dereference_protected(p, lockdep_rtnl_is_held()) +/** + * rcu_replace_pointer_rtnl - replace an RCU pointer under rtnl_lock, returning + * its old value + * @rp: RCU pointer, whose value is returned + * @p: regular pointer + * + * Perform a replacement under rtnl_lock, where @rp is an RCU-annotated + * pointer. The old value of @rp is returned, and @rp is set to @p + */ +#define rcu_replace_pointer_rtnl(rp, p) \ + rcu_replace_pointer(rp, p, lockdep_rtnl_is_held()) + static inline struct netdev_queue *dev_ingress_queue(struct net_device *dev) { return rtnl_dereference(dev->ingress_queue); diff --git a/include/linux/security.h b/include/linux/security.h index 1d1df326c881c7..00809d2d5c38cc 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -32,6 +32,7 @@ #include #include #include +#include struct linux_binprm; struct cred; @@ -2020,15 +2021,22 @@ static inline void securityfs_remove(struct dentry *dentry) union bpf_attr; struct bpf_map; struct bpf_prog; -struct bpf_prog_aux; +struct bpf_token; #ifdef CONFIG_SECURITY extern int security_bpf(int cmd, union bpf_attr *attr, unsigned int size); extern int security_bpf_map(struct bpf_map *map, fmode_t fmode); extern int security_bpf_prog(struct bpf_prog *prog); -extern int security_bpf_map_alloc(struct bpf_map *map); +extern int security_bpf_map_create(struct bpf_map *map, union bpf_attr *attr, + struct bpf_token *token); extern void security_bpf_map_free(struct bpf_map *map); -extern int security_bpf_prog_alloc(struct bpf_prog_aux *aux); -extern void security_bpf_prog_free(struct bpf_prog_aux *aux); +extern int security_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr, + struct bpf_token *token); +extern void security_bpf_prog_free(struct bpf_prog *prog); +extern int security_bpf_token_create(struct bpf_token *token, union bpf_attr *attr, + struct path *path); +extern void security_bpf_token_free(struct bpf_token *token); +extern int security_bpf_token_cmd(const struct bpf_token *token, enum bpf_cmd cmd); +extern int security_bpf_token_capable(const struct bpf_token *token, int cap); #else static inline int security_bpf(int cmd, union bpf_attr *attr, unsigned int size) @@ -2046,7 +2054,8 @@ static inline int security_bpf_prog(struct bpf_prog *prog) return 0; } -static inline int security_bpf_map_alloc(struct bpf_map *map) +static inline int security_bpf_map_create(struct bpf_map *map, union bpf_attr *attr, + struct bpf_token *token) { return 0; } @@ -2054,13 +2063,33 @@ static inline int security_bpf_map_alloc(struct bpf_map *map) static inline void security_bpf_map_free(struct bpf_map *map) { } -static inline int security_bpf_prog_alloc(struct bpf_prog_aux *aux) +static inline int security_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr, + struct bpf_token *token) { return 0; } -static inline void security_bpf_prog_free(struct bpf_prog_aux *aux) +static inline void security_bpf_prog_free(struct bpf_prog *prog) { } + +static inline int security_bpf_token_create(struct bpf_token *token, union bpf_attr *attr, + struct path *path) +{ + return 0; +} + +static inline void security_bpf_token_free(struct bpf_token *token) +{ } + +static inline int security_bpf_token_cmd(const struct bpf_token *token, enum bpf_cmd cmd) +{ + return 0; +} + +static inline int security_bpf_token_capable(const struct bpf_token *token, int cap) +{ + return 0; +} #endif /* CONFIG_SECURITY */ #endif /* CONFIG_BPF_SYSCALL */ diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 7ce38874dbd1f4..ea5c8ab3ed00d0 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -4247,10 +4247,13 @@ static inline bool __skb_metadata_differs(const struct sk_buff *skb_a, { const void *a = skb_metadata_end(skb_a); const void *b = skb_metadata_end(skb_b); - /* Using more efficient variant than plain call to memcmp(). */ -#if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && BITS_PER_LONG == 64 u64 diffs = 0; + if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) || + BITS_PER_LONG != 64) + goto slow; + + /* Using more efficient variant than plain call to memcmp(). */ switch (meta_len) { #define __it(x, op) (x -= sizeof(u##op)) #define __it_diff(a, b, op) (*(u##op *)__it(a, op)) ^ (*(u##op *)__it(b, op)) @@ -4270,11 +4273,11 @@ static inline bool __skb_metadata_differs(const struct sk_buff *skb_a, fallthrough; case 4: diffs |= __it_diff(a, b, 32); break; + default: +slow: + return memcmp(a - meta_len, b - meta_len, meta_len); } return diffs; -#else - return memcmp(a - meta_len, b - meta_len, meta_len); -#endif } static inline bool skb_metadata_differs(const struct sk_buff *skb_a, diff --git a/include/linux/virtio_vsock.h b/include/linux/virtio_vsock.h index ebb3ce63d64dac..c82089dee0c831 100644 --- a/include/linux/virtio_vsock.h +++ b/include/linux/virtio_vsock.h @@ -256,4 +256,5 @@ void virtio_transport_put_credit(struct virtio_vsock_sock *vvs, u32 credit); void virtio_transport_deliver_tap_pkt(struct sk_buff *skb); int virtio_transport_purge_skbs(void *vsk, struct sk_buff_head *list); int virtio_transport_read_skb(struct vsock_sock *vsk, skb_read_actor_t read_actor); +int virtio_transport_notify_set_rcvlowat(struct vsock_sock *vsk, int val); #endif /* _LINUX_VIRTIO_VSOCK_H */ diff --git a/include/net/af_vsock.h b/include/net/af_vsock.h index e302c0e804d0f2..535701efc1e5ce 100644 --- a/include/net/af_vsock.h +++ b/include/net/af_vsock.h @@ -137,7 +137,6 @@ struct vsock_transport { u64 (*stream_rcvhiwat)(struct vsock_sock *); bool (*stream_is_active)(struct vsock_sock *); bool (*stream_allow)(u32 cid, u32 port); - int (*set_rcvlowat)(struct vsock_sock *vsk, int val); /* SEQ_PACKET. */ ssize_t (*seqpacket_dequeue)(struct vsock_sock *vsk, struct msghdr *msg, @@ -168,6 +167,7 @@ struct vsock_transport { struct vsock_transport_send_notify_data *); /* sk_lock held by the caller */ void (*notify_buffer_size)(struct vsock_sock *, u64 *); + int (*notify_set_rcvlowat)(struct vsock_sock *vsk, int val); /* Shutdown. */ int (*shutdown)(struct vsock_sock *, int); diff --git a/include/net/cfg80211.h b/include/net/cfg80211.h index 1b3345129a9549..ac1fb326dcdae6 100644 --- a/include/net/cfg80211.h +++ b/include/net/cfg80211.h @@ -52,7 +52,7 @@ * such wiphy can have zero, one, or many virtual interfaces associated with * it, which need to be identified as such by pointing the network interface's * @ieee80211_ptr pointer to a &struct wireless_dev which further describes - * the wireless part of the interface, normally this struct is embedded in the + * the wireless part of the interface. Normally this struct is embedded in the * network interface's private data area. Drivers can optionally allow creating * or destroying virtual interfaces on the fly, but without at least one or the * ability to create some the wireless device isn't useful. @@ -976,6 +976,15 @@ const struct cfg80211_chan_def * cfg80211_chandef_compatible(const struct cfg80211_chan_def *chandef1, const struct cfg80211_chan_def *chandef2); +/** + * nl80211_chan_width_to_mhz - get the channel width in MHz + * @chan_width: the channel width from &enum nl80211_chan_width + * + * Return: channel width in MHz if the chan_width from &enum nl80211_chan_width + * is valid. -1 otherwise. + */ +int nl80211_chan_width_to_mhz(enum nl80211_chan_width chan_width); + /** * cfg80211_chandef_valid - check if a channel definition is valid * @chandef: the channel definition to check @@ -1664,6 +1673,21 @@ struct link_station_del_parameters { u32 link_id; }; +/** + * struct cfg80211_ttlm_params: TID to link mapping parameters + * + * Used for setting a TID to link mapping. + * + * @dlink: Downlink TID to link mapping, as defined in section 9.4.2.314 + * (TID-To-Link Mapping element) in Draft P802.11be_D4.0. + * @ulink: Uplink TID to link mapping, as defined in section 9.4.2.314 + * (TID-To-Link Mapping element) in Draft P802.11be_D4.0. + */ +struct cfg80211_ttlm_params { + u16 dlink[8]; + u16 ulink[8]; +}; + /** * struct station_parameters - station parameters * @@ -2560,7 +2584,7 @@ struct cfg80211_scan_info { * @short_ssid: short ssid to scan for * @bssid: bssid to scan for * @channel_idx: idx of the channel in the channel array in the scan request - * which the above info relvant to + * which the above info is relevant to * @unsolicited_probe: the AP transmits unsolicited probe response every 20 TU * @short_ssid_valid: @short_ssid is valid and can be used * @psc_no_listen: when set, and the channel is a PSC channel, no need to wait @@ -2819,6 +2843,13 @@ enum cfg80211_signal_type { * the BSS that requested the scan in which the beacon/probe was received. * @chains: bitmask for filled values in @chain_signal. * @chain_signal: per-chain signal strength of last received BSS in dBm. + * @restrict_use: restrict usage, if not set, assume @use_for is + * %NL80211_BSS_USE_FOR_NORMAL. + * @use_for: bitmap of possible usage for this BSS, see + * &enum nl80211_bss_use_for + * @cannot_use_reasons: the reasons (bitmap) for not being able to connect, + * if @restrict_use is set and @use_for is zero (empty); may be 0 for + * unspecified reasons; see &enum nl80211_bss_cannot_use_reasons * @drv_data: Data to be passed through to @inform_bss */ struct cfg80211_inform_bss { @@ -2830,6 +2861,9 @@ struct cfg80211_inform_bss { u8 chains; s8 chain_signal[IEEE80211_MAX_CHAINS]; + u8 restrict_use:1, use_for:7; + u8 cannot_use_reasons; + void *drv_data; }; @@ -2881,6 +2915,11 @@ struct cfg80211_bss_ies { * @chain_signal: per-chain signal strength of last received BSS in dBm. * @bssid_index: index in the multiple BSS set * @max_bssid_indicator: max number of members in the BSS set + * @use_for: bitmap of possible usage for this BSS, see + * &enum nl80211_bss_use_for + * @cannot_use_reasons: the reasons (bitmap) for not being able to connect, + * if @restrict_use is set and @use_for is zero (empty); may be 0 for + * unspecified reasons; see &enum nl80211_bss_cannot_use_reasons * @priv: private area for driver use, has at least wiphy->bss_priv_size bytes */ struct cfg80211_bss { @@ -2906,6 +2945,9 @@ struct cfg80211_bss { u8 bssid_index; u8 max_bssid_indicator; + u8 use_for; + u8 cannot_use_reasons; + u8 priv[] __aligned(sizeof(void *)); }; @@ -4496,6 +4538,7 @@ struct mgmt_frame_regs { * @del_link_station: Remove a link of a station. * * @set_hw_timestamp: Enable/disable HW timestamping of TM/FTM frames. + * @set_ttlm: set the TID to link mapping. */ struct cfg80211_ops { int (*suspend)(struct wiphy *wiphy, struct cfg80211_wowlan *wow); @@ -4855,6 +4898,8 @@ struct cfg80211_ops { struct link_station_del_parameters *params); int (*set_hw_timestamp)(struct wiphy *wiphy, struct net_device *dev, struct cfg80211_set_hw_timestamp *hwts); + int (*set_ttlm)(struct wiphy *wiphy, struct net_device *dev, + struct cfg80211_ttlm_params *params); }; /* @@ -4913,6 +4958,8 @@ struct cfg80211_ops { * NL80211_REGDOM_SET_BY_DRIVER. * @WIPHY_FLAG_CHANNEL_CHANGE_ON_BEACON: reg_call_notifier() is called if driver * set this flag to update channels on beacon hints. + * @WIPHY_FLAG_SUPPORTS_NSTR_NONPRIMARY: support connection to non-primary link + * of an NSTR mobile AP MLD. */ enum wiphy_flags { WIPHY_FLAG_SUPPORTS_EXT_KEK_KCK = BIT(0), @@ -4926,7 +4973,7 @@ enum wiphy_flags { WIPHY_FLAG_IBSS_RSN = BIT(8), WIPHY_FLAG_MESH_AUTH = BIT(10), WIPHY_FLAG_SUPPORTS_EXT_KCK_32 = BIT(11), - /* use hole at 12 */ + WIPHY_FLAG_SUPPORTS_NSTR_NONPRIMARY = BIT(12), WIPHY_FLAG_SUPPORTS_FW_ROAM = BIT(13), WIPHY_FLAG_AP_UAPSD = BIT(14), WIPHY_FLAG_SUPPORTS_TDLS = BIT(15), @@ -7164,6 +7211,25 @@ cfg80211_inform_bss(struct wiphy *wiphy, gfp); } +/** + * __cfg80211_get_bss - get a BSS reference + * @wiphy: the wiphy this BSS struct belongs to + * @channel: the channel to search on (or %NULL) + * @bssid: the desired BSSID (or %NULL) + * @ssid: the desired SSID (or %NULL) + * @ssid_len: length of the SSID (or 0) + * @bss_type: type of BSS, see &enum ieee80211_bss_type + * @privacy: privacy filter, see &enum ieee80211_privacy + * @use_for: indicates which use is intended + */ +struct cfg80211_bss *__cfg80211_get_bss(struct wiphy *wiphy, + struct ieee80211_channel *channel, + const u8 *bssid, + const u8 *ssid, size_t ssid_len, + enum ieee80211_bss_type bss_type, + enum ieee80211_privacy privacy, + u32 use_for); + /** * cfg80211_get_bss - get a BSS reference * @wiphy: the wiphy this BSS struct belongs to @@ -7173,13 +7239,20 @@ cfg80211_inform_bss(struct wiphy *wiphy, * @ssid_len: length of the SSID (or 0) * @bss_type: type of BSS, see &enum ieee80211_bss_type * @privacy: privacy filter, see &enum ieee80211_privacy + * + * This version implies regular usage, %NL80211_BSS_USE_FOR_NORMAL. */ -struct cfg80211_bss *cfg80211_get_bss(struct wiphy *wiphy, - struct ieee80211_channel *channel, - const u8 *bssid, - const u8 *ssid, size_t ssid_len, - enum ieee80211_bss_type bss_type, - enum ieee80211_privacy privacy); +static inline struct cfg80211_bss * +cfg80211_get_bss(struct wiphy *wiphy, struct ieee80211_channel *channel, + const u8 *bssid, const u8 *ssid, size_t ssid_len, + enum ieee80211_bss_type bss_type, + enum ieee80211_privacy privacy) +{ + return __cfg80211_get_bss(wiphy, channel, bssid, ssid, ssid_len, + bss_type, privacy, + NL80211_BSS_USE_FOR_NORMAL); +} + static inline struct cfg80211_bss * cfg80211_get_ibss(struct wiphy *wiphy, struct ieee80211_channel *channel, @@ -7312,7 +7385,7 @@ struct cfg80211_rx_assoc_resp_data { * This function may sleep. The caller must hold the corresponding wdev's mutex. */ void cfg80211_rx_assoc_resp(struct net_device *dev, - struct cfg80211_rx_assoc_resp_data *data); + const struct cfg80211_rx_assoc_resp_data *data); /** * struct cfg80211_assoc_failure - association failure data @@ -7431,7 +7504,7 @@ void cfg80211_notify_new_peer_candidate(struct net_device *dev, * RFkill integration in cfg80211 is almost invisible to drivers, * as cfg80211 automatically registers an rfkill instance for each * wireless device it knows about. Soft kill is also translated - * into disconnecting and turning all interfaces off, drivers are + * into disconnecting and turning all interfaces off. Drivers are * expected to turn off the device when all interfaces are down. * * However, devices may have a hard RFkill line, in which case they @@ -7479,7 +7552,7 @@ static inline void wiphy_rfkill_stop_polling(struct wiphy *wiphy) * the configuration mechanism. * * A driver supporting vendor commands must register them as an array - * in struct wiphy, with handlers for each one, each command has an + * in struct wiphy, with handlers for each one. Each command has an * OUI and sub command ID to identify it. * * Note that this feature should not be (ab)used to implement protocol @@ -7643,7 +7716,7 @@ static inline void cfg80211_vendor_event(struct sk_buff *skb, gfp_t gfp) * interact with driver-specific tools to aid, for instance, * factory programming. * - * This chapter describes how drivers interact with it, for more + * This chapter describes how drivers interact with it. For more * information see the nl80211 book's chapter on it. */ diff --git a/include/net/ip.h b/include/net/ip.h index b31be912489af8..de0c69c57e3cb7 100644 --- a/include/net/ip.h +++ b/include/net/ip.h @@ -356,7 +356,7 @@ static inline void inet_get_local_port_range(const struct net *net, int *low, in *low = range & 0xffff; *high = range >> 16; } -void inet_sk_get_local_port_range(const struct sock *sk, int *low, int *high); +bool inet_sk_get_local_port_range(const struct sock *sk, int *low, int *high); #ifdef CONFIG_SYSCTL static inline bool inet_is_local_reserved_port(struct net *net, unsigned short port) diff --git a/include/net/mac80211.h b/include/net/mac80211.h index 580781ff9dcf6e..77a71b1396b195 100644 --- a/include/net/mac80211.h +++ b/include/net/mac80211.h @@ -2686,6 +2686,9 @@ struct ieee80211_txq { * @IEEE80211_HW_MLO_MCAST_MULTI_LINK_TX: Hardware/driver handles transmitting * multicast frames on all links, mac80211 should not do that. * + * @IEEE80211_HW_DISALLOW_PUNCTURING: HW requires disabling puncturing in EHT + * and connecting with a lower bandwidth instead + * * @NUM_IEEE80211_HW_FLAGS: number of hardware flags, used for sizing arrays */ enum ieee80211_hw_flags { @@ -2743,6 +2746,7 @@ enum ieee80211_hw_flags { IEEE80211_HW_SUPPORTS_CONC_MON_RX_DECAP, IEEE80211_HW_DETECTS_COLOR_COLLISION, IEEE80211_HW_MLO_MCAST_MULTI_LINK_TX, + IEEE80211_HW_DISALLOW_PUNCTURING, /* keep last, obviously */ NUM_IEEE80211_HW_FLAGS @@ -5809,12 +5813,11 @@ void ieee80211_set_key_rx_seq(struct ieee80211_key_conf *keyconf, * ieee80211_remove_key - remove the given key * @keyconf: the parameter passed with the set key * + * Context: Must be called with the wiphy mutex held. + * * Remove the given key. If the key was uploaded to the hardware at the * time this function is called, it is not deleted in the hardware but * instead assumed to have been removed already. - * - * Note that due to locking considerations this function can (currently) - * only be called during key iteration (ieee80211_iter_keys().) */ void ieee80211_remove_key(struct ieee80211_key_conf *keyconf); @@ -6368,12 +6371,12 @@ ieee80211_txq_airtime_check(struct ieee80211_hw *hw, struct ieee80211_txq *txq); * @iter: iterator function that will be called for each key * @iter_data: custom data to pass to the iterator function * + * Context: Must be called with wiphy mutex held; can sleep. + * * This function can be used to iterate all the keys known to * mac80211, even those that weren't previously programmed into * the device. This is intended for use in WoWLAN if the device - * needs reprogramming of the keys during suspend. Note that due - * to locking reasons, it is also only safe to call this at few - * spots since it must hold the RTNL and be able to sleep. + * needs reprogramming of the keys during suspend. * * The order in which the keys are iterated matches the order * in which they were originally installed and handed to the @@ -7435,6 +7438,9 @@ static inline bool ieee80211_is_tx_data(struct sk_buff *skb) * @vif: interface to set active links on * @active_links: the new active links bitmap * + * Context: Must be called with wiphy mutex held; may sleep; calls + * back into the driver. + * * This changes the active links on an interface. The interface * must be in client mode (in AP mode, all links are always active), * and @active_links must be a subset of the vif's valid_links. @@ -7442,6 +7448,7 @@ static inline bool ieee80211_is_tx_data(struct sk_buff *skb) * If a link is switched off and another is switched on at the same * time (e.g. active_links going from 0x1 to 0x10) then you will get * a sequence of calls like + * * - change_vif_links(0x11) * - unassign_vif_chanctx(link_id=0) * - change_sta_links(0x11) for each affected STA (the AP) @@ -7451,10 +7458,6 @@ static inline bool ieee80211_is_tx_data(struct sk_buff *skb) * - change_sta_links(0x10) for each affected STA (the AP) * - assign_vif_chanctx(link_id=4) * - change_vif_links(0x10) - * - * Note: This function acquires some mac80211 locks and must not - * be called with any driver locks held that could cause a - * lock dependency inversion. Best call it without locks. */ int ieee80211_set_active_links(struct ieee80211_vif *vif, u16 active_links); diff --git a/include/net/mana/gdma.h b/include/net/mana/gdma.h index 88b6ef7ce1a6ef..76f2fd2645b73b 100644 --- a/include/net/mana/gdma.h +++ b/include/net/mana/gdma.h @@ -293,6 +293,7 @@ struct gdma_queue { u32 head; u32 tail; + struct list_head entry; /* Extra fields specific to EQ/CQ. */ union { @@ -328,6 +329,7 @@ struct gdma_queue_spec { void *context; unsigned long log2_throttle_limit; + unsigned int msix_index; } eq; struct { @@ -344,7 +346,9 @@ struct gdma_queue_spec { struct gdma_irq_context { void (*handler)(void *arg); - void *arg; + /* Protect the eq_list */ + spinlock_t lock; + struct list_head eq_list; char name[MANA_IRQ_NAME_SZ]; }; @@ -355,7 +359,6 @@ struct gdma_context { unsigned int max_num_queues; unsigned int max_num_msix; unsigned int num_msix_usable; - struct gdma_resource msix_resource; struct gdma_irq_context *irq_contexts; /* L2 MTU */ diff --git a/include/net/netns/core.h b/include/net/netns/core.h index a91ef9f8de60bb..78214f1b43a201 100644 --- a/include/net/netns/core.h +++ b/include/net/netns/core.h @@ -13,6 +13,7 @@ struct netns_core { struct ctl_table_header *sysctl_hdr; int sysctl_somaxconn; + int sysctl_optmem_max; u8 sysctl_txrehash; #ifdef CONFIG_PROC_FS diff --git a/include/net/page_pool/helpers.h b/include/net/page_pool/helpers.h index ead2c0d24b2c71..1d397c1a004318 100644 --- a/include/net/page_pool/helpers.h +++ b/include/net/page_pool/helpers.h @@ -277,6 +277,11 @@ static inline long page_pool_unref_page(struct page *page, long nr) return ret; } +static inline void page_pool_ref_page(struct page *page) +{ + atomic_long_inc(&page->pp_ref_count); +} + static inline bool page_pool_is_last_ref(struct page *page) { /* If page_pool_unref_page() returns 0, we were the last user */ diff --git a/include/net/sock.h b/include/net/sock.h index 1d6931caf0c3c1..8b6fe164b218dc 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -2920,7 +2920,6 @@ extern __u32 sysctl_wmem_max; extern __u32 sysctl_rmem_max; extern int sysctl_tstamp_allow_data; -extern int sysctl_optmem_max; extern __u32 sysctl_wmem_default; extern __u32 sysctl_rmem_default; diff --git a/include/net/xdp.h b/include/net/xdp.h index 349c36fb5fd8f7..e6770dd40c917e 100644 --- a/include/net/xdp.h +++ b/include/net/xdp.h @@ -16,7 +16,7 @@ * * The XDP RX-queue info (xdp_rxq_info) is associated with the driver * level RX-ring queues. It is information that is specific to how - * the driver have configured a given RX-ring queue. + * the driver has configured a given RX-ring queue. * * Each xdp_buff frame received in the driver carries a (pointer) * reference to this xdp_rxq_info structure. This provides the XDP @@ -32,7 +32,7 @@ * The struct is not directly tied to the XDP prog. A new XDP prog * can be attached as long as it doesn't change the underlying * RX-ring. If the RX-ring does change significantly, the NIC driver - * naturally need to stop the RX-ring before purging and reallocating + * naturally needs to stop the RX-ring before purging and reallocating * memory. In that process the driver MUST call unregister (which * also applies for driver shutdown and unload). The register API is * also mandatory during RX-ring setup. @@ -369,7 +369,12 @@ xdp_data_meta_unsupported(const struct xdp_buff *xdp) static inline bool xdp_metalen_invalid(unsigned long metalen) { - return (metalen & (sizeof(__u32) - 1)) || (metalen > 32); + unsigned long meta_max; + + meta_max = type_max(typeof_member(struct skb_shared_info, meta_len)); + BUILD_BUG_ON(!__builtin_constant_p(meta_max)); + + return !IS_ALIGNED(metalen, sizeof(u32)) || metalen > meta_max; } struct xdp_attachment_info { @@ -399,6 +404,10 @@ void xdp_attachment_setup(struct xdp_attachment_info *info, NETDEV_XDP_RX_METADATA_HASH, \ bpf_xdp_metadata_rx_hash, \ xmo_rx_hash) \ + XDP_METADATA_KFUNC(XDP_METADATA_KFUNC_RX_VLAN_TAG, \ + NETDEV_XDP_RX_METADATA_VLAN_TAG, \ + bpf_xdp_metadata_rx_vlan_tag, \ + xmo_rx_vlan_tag) \ enum xdp_rx_metadata { #define XDP_METADATA_KFUNC(name, _, __, ___) name, @@ -427,6 +436,7 @@ enum xdp_rss_hash_type { XDP_RSS_L4_UDP = BIT(5), XDP_RSS_L4_SCTP = BIT(6), XDP_RSS_L4_IPSEC = BIT(7), /* L4 based hash include IPSEC SPI */ + XDP_RSS_L4_ICMP = BIT(8), /* Second part: RSS hash type combinations used for driver HW mapping */ XDP_RSS_TYPE_NONE = 0, @@ -442,11 +452,13 @@ enum xdp_rss_hash_type { XDP_RSS_TYPE_L4_IPV4_UDP = XDP_RSS_L3_IPV4 | XDP_RSS_L4 | XDP_RSS_L4_UDP, XDP_RSS_TYPE_L4_IPV4_SCTP = XDP_RSS_L3_IPV4 | XDP_RSS_L4 | XDP_RSS_L4_SCTP, XDP_RSS_TYPE_L4_IPV4_IPSEC = XDP_RSS_L3_IPV4 | XDP_RSS_L4 | XDP_RSS_L4_IPSEC, + XDP_RSS_TYPE_L4_IPV4_ICMP = XDP_RSS_L3_IPV4 | XDP_RSS_L4 | XDP_RSS_L4_ICMP, XDP_RSS_TYPE_L4_IPV6_TCP = XDP_RSS_L3_IPV6 | XDP_RSS_L4 | XDP_RSS_L4_TCP, XDP_RSS_TYPE_L4_IPV6_UDP = XDP_RSS_L3_IPV6 | XDP_RSS_L4 | XDP_RSS_L4_UDP, XDP_RSS_TYPE_L4_IPV6_SCTP = XDP_RSS_L3_IPV6 | XDP_RSS_L4 | XDP_RSS_L4_SCTP, XDP_RSS_TYPE_L4_IPV6_IPSEC = XDP_RSS_L3_IPV6 | XDP_RSS_L4 | XDP_RSS_L4_IPSEC, + XDP_RSS_TYPE_L4_IPV6_ICMP = XDP_RSS_L3_IPV6 | XDP_RSS_L4 | XDP_RSS_L4_ICMP, XDP_RSS_TYPE_L4_IPV6_TCP_EX = XDP_RSS_TYPE_L4_IPV6_TCP | XDP_RSS_L3_DYNHDR, XDP_RSS_TYPE_L4_IPV6_UDP_EX = XDP_RSS_TYPE_L4_IPV6_UDP | XDP_RSS_L3_DYNHDR, @@ -457,6 +469,8 @@ struct xdp_metadata_ops { int (*xmo_rx_timestamp)(const struct xdp_md *ctx, u64 *timestamp); int (*xmo_rx_hash)(const struct xdp_md *ctx, u32 *hash, enum xdp_rss_hash_type *rss_type); + int (*xmo_rx_vlan_tag)(const struct xdp_md *ctx, __be16 *vlan_proto, + u16 *vlan_tci); }; #ifdef CONFIG_NET diff --git a/include/net/xdp_sock_drv.h b/include/net/xdp_sock_drv.h index 81e02de3f45377..b62bb8525a5fda 100644 --- a/include/net/xdp_sock_drv.h +++ b/include/net/xdp_sock_drv.h @@ -14,6 +14,12 @@ #ifdef CONFIG_XDP_SOCKETS +struct xsk_cb_desc { + void *src; + u8 off; + u8 bytes; +}; + void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries); bool xsk_tx_peek_desc(struct xsk_buff_pool *pool, struct xdp_desc *desc); u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 max); @@ -47,6 +53,12 @@ static inline void xsk_pool_set_rxq_info(struct xsk_buff_pool *pool, xp_set_rxq_info(pool, rxq); } +static inline void xsk_pool_fill_cb(struct xsk_buff_pool *pool, + struct xsk_cb_desc *desc) +{ + xp_fill_cb(pool, desc); +} + static inline unsigned int xsk_pool_get_napi_id(struct xsk_buff_pool *pool) { #ifdef CONFIG_NET_RX_BUSY_POLL @@ -274,6 +286,11 @@ static inline void xsk_pool_set_rxq_info(struct xsk_buff_pool *pool, { } +static inline void xsk_pool_fill_cb(struct xsk_buff_pool *pool, + struct xsk_cb_desc *desc) +{ +} + static inline unsigned int xsk_pool_get_napi_id(struct xsk_buff_pool *pool) { return 0; diff --git a/include/net/xfrm.h b/include/net/xfrm.h index c9bb0f892f5559..1d107241b90187 100644 --- a/include/net/xfrm.h +++ b/include/net/xfrm.h @@ -2190,4 +2190,13 @@ static inline int register_xfrm_interface_bpf(void) #endif +#if IS_ENABLED(CONFIG_DEBUG_INFO_BTF) +int register_xfrm_state_bpf(void); +#else +static inline int register_xfrm_state_bpf(void) +{ + return 0; +} +#endif + #endif /* _NET_XFRM_H */ diff --git a/include/net/xsk_buff_pool.h b/include/net/xsk_buff_pool.h index 8d48d37ab7c0f6..99dd7376df6a7d 100644 --- a/include/net/xsk_buff_pool.h +++ b/include/net/xsk_buff_pool.h @@ -12,6 +12,7 @@ struct xsk_buff_pool; struct xdp_rxq_info; +struct xsk_cb_desc; struct xsk_queue; struct xdp_desc; struct xdp_umem; @@ -135,6 +136,7 @@ static inline void xp_init_xskb_dma(struct xdp_buff_xsk *xskb, struct xsk_buff_p /* AF_XDP ZC drivers, via xdp_sock_buff.h */ void xp_set_rxq_info(struct xsk_buff_pool *pool, struct xdp_rxq_info *rxq); +void xp_fill_cb(struct xsk_buff_pool *pool, struct xsk_cb_desc *desc); int xp_dma_map(struct xsk_buff_pool *pool, struct device *dev, unsigned long attrs, struct page **pages, u32 nr_pages); void xp_dma_unmap(struct xsk_buff_pool *pool, unsigned long attrs); diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index b1e8c5bdfc8242..42f4d3090efe15 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -847,6 +847,36 @@ union bpf_iter_link_info { * Returns zero on success. On error, -1 is returned and *errno* * is set appropriately. * + * BPF_TOKEN_CREATE + * Description + * Create BPF token with embedded information about what + * BPF-related functionality it allows: + * - a set of allowed bpf() syscall commands; + * - a set of allowed BPF map types to be created with + * BPF_MAP_CREATE command, if BPF_MAP_CREATE itself is allowed; + * - a set of allowed BPF program types and BPF program attach + * types to be loaded with BPF_PROG_LOAD command, if + * BPF_PROG_LOAD itself is allowed. + * + * BPF token is created (derived) from an instance of BPF FS, + * assuming it has necessary delegation mount options specified. + * This BPF token can be passed as an extra parameter to various + * bpf() syscall commands to grant BPF subsystem functionality to + * unprivileged processes. + * + * When created, BPF token is "associated" with the owning + * user namespace of BPF FS instance (super block) that it was + * derived from, and subsequent BPF operations performed with + * BPF token would be performing capabilities checks (i.e., + * CAP_BPF, CAP_PERFMON, CAP_NET_ADMIN, CAP_SYS_ADMIN) within + * that user namespace. Without BPF token, such capabilities + * have to be granted in init user namespace, making bpf() + * syscall incompatible with user namespace, for the most part. + * + * Return + * A new file descriptor (a nonnegative integer), or -1 if an + * error occurred (in which case, *errno* is set appropriately). + * * NOTES * eBPF objects (maps and programs) can be shared between processes. * @@ -901,6 +931,8 @@ enum bpf_cmd { BPF_ITER_CREATE, BPF_LINK_DETACH, BPF_PROG_BIND_MAP, + BPF_TOKEN_CREATE, + __MAX_BPF_CMD, }; enum bpf_map_type { @@ -951,6 +983,7 @@ enum bpf_map_type { BPF_MAP_TYPE_BLOOM_FILTER, BPF_MAP_TYPE_USER_RINGBUF, BPF_MAP_TYPE_CGRP_STORAGE, + __MAX_BPF_MAP_TYPE }; /* Note that tracing related programs such as @@ -995,6 +1028,7 @@ enum bpf_prog_type { BPF_PROG_TYPE_SK_LOOKUP, BPF_PROG_TYPE_SYSCALL, /* a program that can execute syscalls */ BPF_PROG_TYPE_NETFILTER, + __MAX_BPF_PROG_TYPE }; enum bpf_attach_type { @@ -1074,9 +1108,11 @@ enum bpf_link_type { BPF_LINK_TYPE_TCX = 11, BPF_LINK_TYPE_UPROBE_MULTI = 12, BPF_LINK_TYPE_NETKIT = 13, - MAX_BPF_LINK_TYPE, + __MAX_BPF_LINK_TYPE, }; +#define MAX_BPF_LINK_TYPE __MAX_BPF_LINK_TYPE + enum bpf_perf_event_type { BPF_PERF_EVENT_UNSPEC = 0, BPF_PERF_EVENT_UPROBE = 1, @@ -1401,6 +1437,7 @@ union bpf_attr { * to using 5 hash functions). */ __u64 map_extra; + __u32 map_token_fd; }; struct { /* anonymous struct used by BPF_MAP_*_ELEM commands */ @@ -1470,6 +1507,7 @@ union bpf_attr { * truncated), or smaller (if log buffer wasn't filled completely). */ __u32 log_true_size; + __u32 prog_token_fd; }; struct { /* anonymous struct used by BPF_OBJ_* commands */ @@ -1582,6 +1620,7 @@ union bpf_attr { * truncated), or smaller (if log buffer wasn't filled completely). */ __u32 btf_log_true_size; + __u32 btf_token_fd; }; struct { @@ -1712,6 +1751,11 @@ union bpf_attr { __u32 flags; /* extra flags */ } prog_bind_map; + struct { /* struct used by BPF_TOKEN_CREATE command */ + __u32 flags; + __u32 bpffs_fd; + } token_create; + } __attribute__((aligned(8))); /* The description below is an attempt at providing documentation to eBPF diff --git a/include/uapi/linux/mptcp_pm.h b/include/uapi/linux/mptcp_pm.h index b5d11aece4089d..50589e5dd6a38a 100644 --- a/include/uapi/linux/mptcp_pm.h +++ b/include/uapi/linux/mptcp_pm.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */ /* Do not edit directly, auto-generated from: */ -/* Documentation/netlink/specs/mptcp.yaml */ +/* Documentation/netlink/specs/mptcp_pm.yaml */ /* YNL-GEN uapi header */ #ifndef _UAPI_LINUX_MPTCP_PM_H diff --git a/include/uapi/linux/netdev.h b/include/uapi/linux/netdev.h index 424c5e28f49519..93cb411adf72e4 100644 --- a/include/uapi/linux/netdev.h +++ b/include/uapi/linux/netdev.h @@ -44,10 +44,13 @@ enum netdev_xdp_act { * timestamp via bpf_xdp_metadata_rx_timestamp(). * @NETDEV_XDP_RX_METADATA_HASH: Device is capable of exposing receive packet * hash via bpf_xdp_metadata_rx_hash(). + * @NETDEV_XDP_RX_METADATA_VLAN_TAG: Device is capable of exposing receive + * packet VLAN tag via bpf_xdp_metadata_rx_vlan_tag(). */ enum netdev_xdp_rx_metadata { NETDEV_XDP_RX_METADATA_TIMESTAMP = 1, NETDEV_XDP_RX_METADATA_HASH = 2, + NETDEV_XDP_RX_METADATA_VLAN_TAG = 4, }; /** diff --git a/include/uapi/linux/nl80211.h b/include/uapi/linux/nl80211.h index 0cd1da2c2902a9..a682b54bd3bade 100644 --- a/include/uapi/linux/nl80211.h +++ b/include/uapi/linux/nl80211.h @@ -72,7 +72,7 @@ * For drivers supporting TDLS with external setup (WIPHY_FLAG_SUPPORTS_TDLS * and WIPHY_FLAG_TDLS_EXTERNAL_SETUP), the station lifetime is as follows: * - a setup station entry is added, not yet authorized, without any rate - * or capability information, this just exists to avoid race conditions + * or capability information; this just exists to avoid race conditions * - when the TDLS setup is done, a single NL80211_CMD_SET_STATION is valid * to add rate and capability information to the station and at the same * time mark it authorized. @@ -87,7 +87,7 @@ * DOC: Frame transmission/registration support * * Frame transmission and registration support exists to allow userspace - * management entities such as wpa_supplicant react to management frames + * management entities such as wpa_supplicant to react to management frames * that are not being handled by the kernel. This includes, for example, * certain classes of action frames that cannot be handled in the kernel * for various reasons. @@ -113,7 +113,7 @@ * * Frame transmission allows userspace to send for example the required * responses to action frames. It is subject to some sanity checking, - * but many frames can be transmitted. When a frame was transmitted, its + * but many frames can be transmitted. When a frame is transmitted, its * status is indicated to the sending socket. * * For more technical details, see the corresponding command descriptions @@ -123,7 +123,7 @@ /** * DOC: Virtual interface / concurrency capabilities * - * Some devices are able to operate with virtual MACs, they can have + * Some devices are able to operate with virtual MACs; they can have * more than one virtual interface. The capability handling for this * is a bit complex though, as there may be a number of restrictions * on the types of concurrency that are supported. @@ -135,7 +135,7 @@ * Once concurrency is desired, more attributes must be observed: * To start with, since some interface types are purely managed in * software, like the AP-VLAN type in mac80211 for example, there's - * an additional list of these, they can be added at any time and + * an additional list of these; they can be added at any time and * are only restricted by some semantic restrictions (e.g. AP-VLAN * cannot be added without a corresponding AP interface). This list * is exported in the %NL80211_ATTR_SOFTWARE_IFTYPES attribute. @@ -164,7 +164,7 @@ * Packet coalesce feature helps to reduce number of received interrupts * to host by buffering these packets in firmware/hardware for some * predefined time. Received interrupt will be generated when one of the - * following events occur. + * following events occurs. * a) Expiration of hardware timer whose expiration time is set to maximum * coalescing delay of matching coalesce rule. * b) Coalescing buffer in hardware reaches its limit. @@ -174,7 +174,7 @@ * rule. * a) Maximum coalescing delay * b) List of packet patterns which needs to be matched - * c) Condition for coalescence. pattern 'match' or 'no match' + * c) Condition for coalescence: pattern 'match' or 'no match' * Multiple such rules can be created. */ @@ -213,7 +213,7 @@ /** * DOC: FILS shared key authentication offload * - * FILS shared key authentication offload can be advertized by drivers by + * FILS shared key authentication offload can be advertised by drivers by * setting @NL80211_EXT_FEATURE_FILS_SK_OFFLOAD flag. The drivers that support * FILS shared key authentication offload should be able to construct the * authentication and association frames for FILS shared key authentication and @@ -239,7 +239,7 @@ * The PMKSA can be maintained in userspace persistently so that it can be used * later after reboots or wifi turn off/on also. * - * %NL80211_ATTR_FILS_CACHE_ID is the cache identifier advertized by a FILS + * %NL80211_ATTR_FILS_CACHE_ID is the cache identifier advertised by a FILS * capable AP supporting PMK caching. It specifies the scope within which the * PMKSAs are cached in an ESS. %NL80211_CMD_SET_PMKSA and * %NL80211_CMD_DEL_PMKSA are enhanced to allow support for PMKSA caching based @@ -290,12 +290,12 @@ * If the configuration needs to be applied for specific peer then the MAC * address of the peer needs to be passed in %NL80211_ATTR_MAC, otherwise the * configuration will be applied for all the connected peers in the vif except - * any peers that have peer specific configuration for the TID by default; if - * the %NL80211_TID_CONFIG_ATTR_OVERRIDE flag is set, peer specific values + * any peers that have peer-specific configuration for the TID by default; if + * the %NL80211_TID_CONFIG_ATTR_OVERRIDE flag is set, peer-specific values * will be overwritten. * - * All this configuration is valid only for STA's current connection - * i.e. the configuration will be reset to default when the STA connects back + * All this configuration is valid only for STA's current connection, + * i.e., the configuration will be reset to default when the STA connects back * after disconnection/roaming, and this configuration will be cleared when * the interface goes down. */ @@ -521,7 +521,7 @@ * %NL80211_ATTR_SCHED_SCAN_PLANS. If %NL80211_ATTR_SCHED_SCAN_PLANS is * not specified and only %NL80211_ATTR_SCHED_SCAN_INTERVAL is specified, * scheduled scan will run in an infinite loop with the specified interval. - * These attributes are mutually exculsive, + * These attributes are mutually exclusive, * i.e. NL80211_ATTR_SCHED_SCAN_INTERVAL must not be passed if * NL80211_ATTR_SCHED_SCAN_PLANS is defined. * If for some reason scheduled scan is aborted by the driver, all scan @@ -552,7 +552,7 @@ * %NL80211_CMD_STOP_SCHED_SCAN command is received or when the interface * is brought down while a scheduled scan was running. * - * @NL80211_CMD_GET_SURVEY: get survey resuls, e.g. channel occupation + * @NL80211_CMD_GET_SURVEY: get survey results, e.g. channel occupation * or noise level * @NL80211_CMD_NEW_SURVEY_RESULTS: survey data notification (as a reply to * NL80211_CMD_GET_SURVEY and on the "scan" multicast group) @@ -563,12 +563,13 @@ * using %NL80211_ATTR_SSID, %NL80211_ATTR_FILS_CACHE_ID, * %NL80211_ATTR_PMKID, and %NL80211_ATTR_PMK in case of FILS * authentication where %NL80211_ATTR_FILS_CACHE_ID is the identifier - * advertized by a FILS capable AP identifying the scope of PMKSA in an + * advertised by a FILS capable AP identifying the scope of PMKSA in an * ESS. * @NL80211_CMD_DEL_PMKSA: Delete a PMKSA cache entry, using %NL80211_ATTR_MAC * (for the BSSID) and %NL80211_ATTR_PMKID or using %NL80211_ATTR_SSID, * %NL80211_ATTR_FILS_CACHE_ID, and %NL80211_ATTR_PMKID in case of FILS - * authentication. + * authentication. Additionally in case of SAE offload and OWE offloads + * PMKSA entry can be deleted using %NL80211_ATTR_SSID. * @NL80211_CMD_FLUSH_PMKSA: Flush all PMKSA cache entries. * * @NL80211_CMD_REG_CHANGE: indicates to userspace the regulatory domain @@ -607,7 +608,7 @@ * BSSID in case of station mode). %NL80211_ATTR_SSID is used to specify * the SSID (mainly for association, but is included in authentication * request, too, to help BSS selection. %NL80211_ATTR_WIPHY_FREQ + - * %NL80211_ATTR_WIPHY_FREQ_OFFSET is used to specify the frequence of the + * %NL80211_ATTR_WIPHY_FREQ_OFFSET is used to specify the frequency of the * channel in MHz. %NL80211_ATTR_AUTH_TYPE is used to specify the * authentication type. %NL80211_ATTR_IE is used to define IEs * (VendorSpecificInfo, but also including RSN IE and FT IEs) to be added @@ -816,7 +817,7 @@ * reached. * @NL80211_CMD_SET_CHANNEL: Set the channel (using %NL80211_ATTR_WIPHY_FREQ * and the attributes determining channel width) the given interface - * (identifed by %NL80211_ATTR_IFINDEX) shall operate on. + * (identified by %NL80211_ATTR_IFINDEX) shall operate on. * In case multiple channels are supported by the device, the mechanism * with which it switches channels is implementation-defined. * When a monitor interface is given, it can only switch channel while @@ -888,7 +889,7 @@ * inform userspace of the new replay counter. * * @NL80211_CMD_PMKSA_CANDIDATE: This is used as an event to inform userspace - * of PMKSA caching dandidates. + * of PMKSA caching candidates. * * @NL80211_CMD_TDLS_OPER: Perform a high-level TDLS command (e.g. link setup). * In addition, this can be used as an event to request userspace to take @@ -924,7 +925,7 @@ * * @NL80211_CMD_PROBE_CLIENT: Probe an associated station on an AP interface * by sending a null data frame to it and reporting when the frame is - * acknowleged. This is used to allow timing out inactive clients. Uses + * acknowledged. This is used to allow timing out inactive clients. Uses * %NL80211_ATTR_IFINDEX and %NL80211_ATTR_MAC. The command returns a * direct reply with an %NL80211_ATTR_COOKIE that is later used to match * up the event with the request. The event includes the same data and @@ -1327,6 +1328,11 @@ * Multi-Link reconfiguration. %NL80211_ATTR_MLO_LINKS is used to provide * information about the removed STA MLD setup links. * + * @NL80211_CMD_SET_TID_TO_LINK_MAPPING: Set the TID to Link Mapping for a + * non-AP MLD station. The %NL80211_ATTR_MLO_TTLM_DLINK and + * %NL80211_ATTR_MLO_TTLM_ULINK attributes are used to specify the + * TID to Link mapping for downlink/uplink traffic. + * * @NL80211_CMD_MAX: highest used command number * @__NL80211_CMD_AFTER_LAST: internal use */ @@ -1582,6 +1588,8 @@ enum nl80211_commands { NL80211_CMD_LINKS_REMOVED, + NL80211_CMD_SET_TID_TO_LINK_MAPPING, + /* add new commands above here */ /* used to define NL80211_CMD_MAX below */ @@ -1839,7 +1847,7 @@ enum nl80211_commands { * using %CMD_CONTROL_PORT_FRAME. If control port routing over NL80211 is * to be used then userspace must also use the %NL80211_ATTR_SOCKET_OWNER * flag. When used with %NL80211_ATTR_CONTROL_PORT_NO_PREAUTH, pre-auth - * frames are not forwared over the control port. + * frames are not forwarded over the control port. * * @NL80211_ATTR_TESTDATA: Testmode data blob, passed through to the driver. * We recommend using nested, driver-specific attributes within this. @@ -1976,10 +1984,10 @@ enum nl80211_commands { * bit. Depending on which antennas are selected in the bitmap, 802.11n * drivers can derive which chainmasks to use (if all antennas belonging to * a particular chain are disabled this chain should be disabled) and if - * a chain has diversity antennas wether diversity should be used or not. + * a chain has diversity antennas whether diversity should be used or not. * HT capabilities (STBC, TX Beamforming, Antenna selection) can be * derived from the available chains after applying the antenna mask. - * Non-802.11n drivers can derive wether to use diversity or not. + * Non-802.11n drivers can derive whether to use diversity or not. * Drivers may reject configurations or RX/TX mask combinations they cannot * support by returning -EINVAL. * @@ -2549,7 +2557,7 @@ enum nl80211_commands { * from successful FILS authentication and is used with * %NL80211_CMD_CONNECT. * - * @NL80211_ATTR_FILS_CACHE_ID: A 2-octet identifier advertized by a FILS AP + * @NL80211_ATTR_FILS_CACHE_ID: A 2-octet identifier advertised by a FILS AP * identifying the scope of PMKSAs. This is used with * @NL80211_CMD_SET_PMKSA and @NL80211_CMD_DEL_PMKSA. * @@ -2830,6 +2838,19 @@ enum nl80211_commands { * @NL80211_ATTR_MLO_LINK_DISABLED: Flag attribute indicating that the link is * disabled. * + * @NL80211_ATTR_BSS_DUMP_INCLUDE_USE_DATA: Include BSS usage data, i.e. + * include BSSes that can only be used in restricted scenarios and/or + * cannot be used at all. + * + * @NL80211_ATTR_MLO_TTLM_DLINK: Binary attribute specifying the downlink TID to + * link mapping. The length is 8 * sizeof(u16). For each TID the link + * mapping is as defined in section 9.4.2.314 (TID-To-Link Mapping element) + * in Draft P802.11be_D4.0. + * @NL80211_ATTR_MLO_TTLM_ULINK: Binary attribute specifying the uplink TID to + * link mapping. The length is 8 * sizeof(u16). For each TID the link + * mapping is as defined in section 9.4.2.314 (TID-To-Link Mapping element) + * in Draft P802.11be_D4.0. + * * @NUM_NL80211_ATTR: total number of nl80211_attrs available * @NL80211_ATTR_MAX: highest attribute number currently defined * @__NL80211_ATTR_AFTER_LAST: internal use @@ -3368,6 +3389,11 @@ enum nl80211_attrs { NL80211_ATTR_MLO_LINK_DISABLED, + NL80211_ATTR_BSS_DUMP_INCLUDE_USE_DATA, + + NL80211_ATTR_MLO_TTLM_DLINK, + NL80211_ATTR_MLO_TTLM_ULINK, + /* add attributes here, update the policy in nl80211.c */ __NL80211_ATTR_AFTER_LAST, @@ -4174,7 +4200,7 @@ enum nl80211_wmm_rule { * (100 * dBm). * @NL80211_FREQUENCY_ATTR_DFS_STATE: current state for DFS * (enum nl80211_dfs_state) - * @NL80211_FREQUENCY_ATTR_DFS_TIME: time in miliseconds for how long + * @NL80211_FREQUENCY_ATTR_DFS_TIME: time in milliseconds for how long * this channel is in this DFS state. * @NL80211_FREQUENCY_ATTR_NO_HT40_MINUS: HT40- isn't possible with this * channel as the control channel @@ -5031,6 +5057,30 @@ enum nl80211_bss_scan_width { NL80211_BSS_CHAN_WIDTH_2, }; +/** + * enum nl80211_bss_use_for - bitmap indicating possible BSS use + * @NL80211_BSS_USE_FOR_NORMAL: Use this BSS for normal "connection", + * including IBSS/MBSS depending on the type. + * @NL80211_BSS_USE_FOR_MLD_LINK: This BSS can be used as a link in an + * MLO connection. Note that for an MLO connection, all links including + * the assoc link must have this flag set, and the assoc link must + * additionally have %NL80211_BSS_USE_FOR_NORMAL set. + */ +enum nl80211_bss_use_for { + NL80211_BSS_USE_FOR_NORMAL = 1 << 0, + NL80211_BSS_USE_FOR_MLD_LINK = 1 << 1, +}; + +/** + * enum nl80211_bss_cannot_use_reasons - reason(s) connection to a + * BSS isn't possible + * @NL80211_BSS_CANNOT_USE_NSTR_NONPRIMARY: NSTR nonprimary links aren't + * supported by the device, and this BSS entry represents one. + */ +enum nl80211_bss_cannot_use_reasons { + NL80211_BSS_CANNOT_USE_NSTR_NONPRIMARY = 1 << 0, +}; + /** * enum nl80211_bss - netlink attributes for a BSS * @@ -5083,6 +5133,14 @@ enum nl80211_bss_scan_width { * @NL80211_BSS_FREQUENCY_OFFSET: frequency offset in KHz * @NL80211_BSS_MLO_LINK_ID: MLO link ID of the BSS (u8). * @NL80211_BSS_MLD_ADDR: MLD address of this BSS if connected to it. + * @NL80211_BSS_USE_FOR: u32 bitmap attribute indicating what the BSS can be + * used for, see &enum nl80211_bss_use_for. + * @NL80211_BSS_CANNOT_USE_REASONS: Indicates the reason that this BSS cannot + * be used for all or some of the possible uses by the device reporting it, + * even though its presence was detected. + * This is a u64 attribute containing a bitmap of values from + * &enum nl80211_cannot_use_reasons, note that the attribute may be missing + * if no reasons are specified. * @__NL80211_BSS_AFTER_LAST: internal * @NL80211_BSS_MAX: highest BSS attribute */ @@ -5110,6 +5168,8 @@ enum nl80211_bss { NL80211_BSS_FREQUENCY_OFFSET, NL80211_BSS_MLO_LINK_ID, NL80211_BSS_MLD_ADDR, + NL80211_BSS_USE_FOR, + NL80211_BSS_CANNOT_USE_REASONS, /* keep last */ __NL80211_BSS_AFTER_LAST, @@ -5458,7 +5518,7 @@ enum nl80211_tx_rate_setting { * (%NL80211_TID_CONFIG_ATTR_TIDS, %NL80211_TID_CONFIG_ATTR_OVERRIDE). * @NL80211_TID_CONFIG_ATTR_PEER_SUPP: same as the previous per-vif one, but * per peer instead. - * @NL80211_TID_CONFIG_ATTR_OVERRIDE: flag attribue, if set indicates + * @NL80211_TID_CONFIG_ATTR_OVERRIDE: flag attribute, if set indicates * that the new configuration overrides all previous peer * configurations, otherwise previous peer specific configurations * should be left untouched. @@ -5841,7 +5901,7 @@ enum nl80211_attr_coalesce_rule { /** * enum nl80211_coalesce_condition - coalesce rule conditions - * @NL80211_COALESCE_CONDITION_MATCH: coalaesce Rx packets when patterns + * @NL80211_COALESCE_CONDITION_MATCH: coalesce Rx packets when patterns * in a rule are matched. * @NL80211_COALESCE_CONDITION_NO_MATCH: coalesce Rx packets when patterns * in a rule are not matched. @@ -5940,7 +6000,7 @@ enum nl80211_if_combination_attrs { * enum nl80211_plink_state - state of a mesh peer link finite state machine * * @NL80211_PLINK_LISTEN: initial state, considered the implicit - * state of non existent mesh peer links + * state of non-existent mesh peer links * @NL80211_PLINK_OPN_SNT: mesh plink open frame has been sent to * this mesh peer * @NL80211_PLINK_OPN_RCVD: mesh plink open frame has been received @@ -6233,7 +6293,7 @@ enum nl80211_feature_flags { * request to use RRM (see %NL80211_ATTR_USE_RRM) with * %NL80211_CMD_ASSOCIATE and %NL80211_CMD_CONNECT requests, which will set * the ASSOC_REQ_USE_RRM flag in the association request even if - * NL80211_FEATURE_QUIET is not advertized. + * NL80211_FEATURE_QUIET is not advertised. * @NL80211_EXT_FEATURE_MU_MIMO_AIR_SNIFFER: This device supports MU-MIMO air * sniffer which means that it can be configured to hear packets from * certain groups which can be configured by the @@ -6245,7 +6305,7 @@ enum nl80211_feature_flags { * the BSS that the interface that requested the scan is connected to * (if available). * @NL80211_EXT_FEATURE_BSS_PARENT_TSF: Per BSS, this driver reports the - * time the last beacon/probe was received. For a non MLO connection, the + * time the last beacon/probe was received. For a non-MLO connection, the * time is the TSF of the BSS that the interface that requested the scan is * connected to (if available). For an MLO connection, the time is the TSF * of the BSS corresponding with link ID specified in the scan request (if @@ -6253,7 +6313,7 @@ enum nl80211_feature_flags { * @NL80211_EXT_FEATURE_SET_SCAN_DWELL: This driver supports configuration of * channel dwell time. * @NL80211_EXT_FEATURE_BEACON_RATE_LEGACY: Driver supports beacon rate - * configuration (AP/mesh), supporting a legacy (non HT/VHT) rate. + * configuration (AP/mesh), supporting a legacy (non-HT/VHT) rate. * @NL80211_EXT_FEATURE_BEACON_RATE_HT: Driver supports beacon rate * configuration (AP/mesh) with HT rates. * @NL80211_EXT_FEATURE_BEACON_RATE_VHT: Driver supports beacon rate @@ -6589,7 +6649,7 @@ enum nl80211_timeout_reason { * request parameters IE in the probe request * @NL80211_SCAN_FLAG_ACCEPT_BCAST_PROBE_RESP: accept broadcast probe responses * @NL80211_SCAN_FLAG_OCE_PROBE_REQ_HIGH_TX_RATE: send probe request frames at - * rate of at least 5.5M. In case non OCE AP is discovered in the channel, + * rate of at least 5.5M. In case non-OCE AP is discovered in the channel, * only the first probe req in the channel will be sent in high rate. * @NL80211_SCAN_FLAG_OCE_PROBE_REQ_DEFERRAL_SUPPRESSION: allow probe request * tx deferral (dot11FILSProbeDelay shall be set to 15ms) @@ -6625,7 +6685,7 @@ enum nl80211_timeout_reason { * received on the 2.4/5 GHz channels to actively scan only the 6GHz * channels on which APs are expected to be found. Note that when not set, * the scan logic would scan all 6GHz channels, but since transmission of - * probe requests on non PSC channels is limited, it is highly likely that + * probe requests on non-PSC channels is limited, it is highly likely that * these channels would passively be scanned. Also note that when the flag * is set, in addition to the colocated APs, PSC channels would also be * scanned if the user space has asked for it. @@ -6957,7 +7017,7 @@ enum nl80211_nan_func_term_reason { * The instance ID for the follow up Service Discovery Frame. This is u8. * @NL80211_NAN_FUNC_FOLLOW_UP_REQ_ID: relevant if the function's type * is follow up. This is a u8. - * The requestor instance ID for the follow up Service Discovery Frame. + * The requester instance ID for the follow up Service Discovery Frame. * @NL80211_NAN_FUNC_FOLLOW_UP_DEST: the MAC address of the recipient of the * follow up Service Discovery Frame. This is a binary attribute. * @NL80211_NAN_FUNC_CLOSE_RANGE: is this function limited for devices in a @@ -7347,7 +7407,7 @@ enum nl80211_peer_measurement_attrs { * @NL80211_PMSR_FTM_CAPA_ATTR_TRIGGER_BASED: flag attribute indicating if * trigger based ranging measurement is supported * @NL80211_PMSR_FTM_CAPA_ATTR_NON_TRIGGER_BASED: flag attribute indicating - * if non trigger based ranging measurement is supported + * if non-trigger-based ranging measurement is supported * * @NUM_NL80211_PMSR_FTM_CAPA_ATTR: internal * @NL80211_PMSR_FTM_CAPA_ATTR_MAX: highest attribute number @@ -7401,7 +7461,7 @@ enum nl80211_peer_measurement_ftm_capa { * if neither %NL80211_PMSR_FTM_REQ_ATTR_TRIGGER_BASED nor * %NL80211_PMSR_FTM_REQ_ATTR_NON_TRIGGER_BASED is set, EDCA based * ranging will be used. - * @NL80211_PMSR_FTM_REQ_ATTR_NON_TRIGGER_BASED: request non trigger based + * @NL80211_PMSR_FTM_REQ_ATTR_NON_TRIGGER_BASED: request non-trigger-based * ranging measurement (flag) * This attribute and %NL80211_PMSR_FTM_REQ_ATTR_TRIGGER_BASED are * mutually exclusive. @@ -7479,7 +7539,7 @@ enum nl80211_peer_measurement_ftm_failure_reasons { * @NL80211_PMSR_FTM_RESP_ATTR_NUM_FTMR_ATTEMPTS: number of FTM Request frames * transmitted (u32, optional) * @NL80211_PMSR_FTM_RESP_ATTR_NUM_FTMR_SUCCESSES: number of FTM Request frames - * that were acknowleged (u32, optional) + * that were acknowledged (u32, optional) * @NL80211_PMSR_FTM_RESP_ATTR_BUSY_RETRY_TIME: retry time received from the * busy peer (u32, seconds) * @NL80211_PMSR_FTM_RESP_ATTR_NUM_BURSTS_EXP: actual number of bursts exponent diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile index f526b7573e97c1..4ce95acfcaa728 100644 --- a/kernel/bpf/Makefile +++ b/kernel/bpf/Makefile @@ -6,7 +6,7 @@ cflags-nogcse-$(CONFIG_X86)$(CONFIG_CC_IS_GCC) := -fno-gcse endif CFLAGS_core.o += $(call cc-disable-warning, override-init) $(cflags-nogcse-yy) -obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o log.o +obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o log.o token.o obj-$(CONFIG_BPF_SYSCALL) += bpf_iter.o map_iter.o task_iter.o prog_iter.o link_iter.o obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o bloom_filter.o obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c index c85ff9162a5cd4..13358675ff2edc 100644 --- a/kernel/bpf/arraymap.c +++ b/kernel/bpf/arraymap.c @@ -82,7 +82,7 @@ static struct bpf_map *array_map_alloc(union bpf_attr *attr) bool percpu = attr->map_type == BPF_MAP_TYPE_PERCPU_ARRAY; int numa_node = bpf_map_attr_numa_node(attr); u32 elem_size, index_mask, max_entries; - bool bypass_spec_v1 = bpf_bypass_spec_v1(); + bool bypass_spec_v1 = bpf_bypass_spec_v1(NULL); u64 array_size, mask64; struct bpf_array *array; @@ -867,11 +867,11 @@ int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file, } if (old_ptr) - map->ops->map_fd_put_ptr(old_ptr); + map->ops->map_fd_put_ptr(map, old_ptr, true); return 0; } -static long fd_array_map_delete_elem(struct bpf_map *map, void *key) +static long __fd_array_map_delete_elem(struct bpf_map *map, void *key, bool need_defer) { struct bpf_array *array = container_of(map, struct bpf_array, map); void *old_ptr; @@ -890,13 +890,18 @@ static long fd_array_map_delete_elem(struct bpf_map *map, void *key) } if (old_ptr) { - map->ops->map_fd_put_ptr(old_ptr); + map->ops->map_fd_put_ptr(map, old_ptr, need_defer); return 0; } else { return -ENOENT; } } +static long fd_array_map_delete_elem(struct bpf_map *map, void *key) +{ + return __fd_array_map_delete_elem(map, key, true); +} + static void *prog_fd_array_get_ptr(struct bpf_map *map, struct file *map_file, int fd) { @@ -913,8 +918,9 @@ static void *prog_fd_array_get_ptr(struct bpf_map *map, return prog; } -static void prog_fd_array_put_ptr(void *ptr) +static void prog_fd_array_put_ptr(struct bpf_map *map, void *ptr, bool need_defer) { + /* bpf_prog is freed after one RCU or tasks trace grace period */ bpf_prog_put(ptr); } @@ -924,13 +930,13 @@ static u32 prog_fd_array_sys_lookup_elem(void *ptr) } /* decrement refcnt of all bpf_progs that are stored in this map */ -static void bpf_fd_array_map_clear(struct bpf_map *map) +static void bpf_fd_array_map_clear(struct bpf_map *map, bool need_defer) { struct bpf_array *array = container_of(map, struct bpf_array, map); int i; for (i = 0; i < array->map.max_entries; i++) - fd_array_map_delete_elem(map, &i); + __fd_array_map_delete_elem(map, &i, need_defer); } static void prog_array_map_seq_show_elem(struct bpf_map *map, void *key, @@ -1071,7 +1077,7 @@ static void prog_array_map_clear_deferred(struct work_struct *work) { struct bpf_map *map = container_of(work, struct bpf_array_aux, work)->map; - bpf_fd_array_map_clear(map); + bpf_fd_array_map_clear(map, true); bpf_map_put(map); } @@ -1151,7 +1157,7 @@ static struct bpf_event_entry *bpf_event_entry_gen(struct file *perf_file, { struct bpf_event_entry *ee; - ee = kzalloc(sizeof(*ee), GFP_ATOMIC); + ee = kzalloc(sizeof(*ee), GFP_KERNEL); if (ee) { ee->event = perf_file->private_data; ee->perf_file = perf_file; @@ -1201,8 +1207,9 @@ static void *perf_event_fd_array_get_ptr(struct bpf_map *map, return ee; } -static void perf_event_fd_array_put_ptr(void *ptr) +static void perf_event_fd_array_put_ptr(struct bpf_map *map, void *ptr, bool need_defer) { + /* bpf_perf_event is freed after one RCU grace period */ bpf_event_entry_free_rcu(ptr); } @@ -1220,7 +1227,7 @@ static void perf_event_fd_array_release(struct bpf_map *map, for (i = 0; i < array->map.max_entries; i++) { ee = READ_ONCE(array->ptrs[i]); if (ee && ee->map_file == map_file) - fd_array_map_delete_elem(map, &i); + __fd_array_map_delete_elem(map, &i, true); } rcu_read_unlock(); } @@ -1228,7 +1235,7 @@ static void perf_event_fd_array_release(struct bpf_map *map, static void perf_event_fd_array_map_free(struct bpf_map *map) { if (map->map_flags & BPF_F_PRESERVE_ELEMS) - bpf_fd_array_map_clear(map); + bpf_fd_array_map_clear(map, false); fd_array_map_free(map); } @@ -1256,7 +1263,7 @@ static void *cgroup_fd_array_get_ptr(struct bpf_map *map, return cgroup_get_from_fd(fd); } -static void cgroup_fd_array_put_ptr(void *ptr) +static void cgroup_fd_array_put_ptr(struct bpf_map *map, void *ptr, bool need_defer) { /* cgroup_put free cgrp after a rcu grace period */ cgroup_put(ptr); @@ -1264,7 +1271,7 @@ static void cgroup_fd_array_put_ptr(void *ptr) static void cgroup_fd_array_free(struct bpf_map *map) { - bpf_fd_array_map_clear(map); + bpf_fd_array_map_clear(map, false); fd_array_map_free(map); } @@ -1309,7 +1316,7 @@ static void array_of_map_free(struct bpf_map *map) * is protected by fdget/fdput. */ bpf_map_meta_free(map->inner_map_meta); - bpf_fd_array_map_clear(map); + bpf_fd_array_map_clear(map, false); fd_array_map_free(map); } diff --git a/kernel/bpf/bpf_cgrp_storage.c b/kernel/bpf/bpf_cgrp_storage.c index d44fe8dd973295..28efd0a3f2200c 100644 --- a/kernel/bpf/bpf_cgrp_storage.c +++ b/kernel/bpf/bpf_cgrp_storage.c @@ -82,7 +82,7 @@ static void *bpf_cgrp_storage_lookup_elem(struct bpf_map *map, void *key) int fd; fd = *(int *)key; - cgroup = cgroup_get_from_fd(fd); + cgroup = cgroup_v1v2_get_from_fd(fd); if (IS_ERR(cgroup)) return ERR_CAST(cgroup); @@ -101,7 +101,7 @@ static long bpf_cgrp_storage_update_elem(struct bpf_map *map, void *key, int fd; fd = *(int *)key; - cgroup = cgroup_get_from_fd(fd); + cgroup = cgroup_v1v2_get_from_fd(fd); if (IS_ERR(cgroup)) return PTR_ERR(cgroup); @@ -131,7 +131,7 @@ static long bpf_cgrp_storage_delete_elem(struct bpf_map *map, void *key) int err, fd; fd = *(int *)key; - cgroup = cgroup_get_from_fd(fd); + cgroup = cgroup_v1v2_get_from_fd(fd); if (IS_ERR(cgroup)) return PTR_ERR(cgroup); diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c index e14c822f8911c9..63b4dc4951253e 100644 --- a/kernel/bpf/bpf_lsm.c +++ b/kernel/bpf/bpf_lsm.c @@ -260,9 +260,15 @@ bpf_lsm_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) BTF_SET_START(sleepable_lsm_hooks) BTF_ID(func, bpf_lsm_bpf) BTF_ID(func, bpf_lsm_bpf_map) -BTF_ID(func, bpf_lsm_bpf_map_alloc_security) -BTF_ID(func, bpf_lsm_bpf_map_free_security) +BTF_ID(func, bpf_lsm_bpf_map_create) +BTF_ID(func, bpf_lsm_bpf_map_free) BTF_ID(func, bpf_lsm_bpf_prog) +BTF_ID(func, bpf_lsm_bpf_prog_load) +BTF_ID(func, bpf_lsm_bpf_prog_free) +BTF_ID(func, bpf_lsm_bpf_token_create) +BTF_ID(func, bpf_lsm_bpf_token_free) +BTF_ID(func, bpf_lsm_bpf_token_cmd) +BTF_ID(func, bpf_lsm_bpf_token_capable) BTF_ID(func, bpf_lsm_bprm_check_security) BTF_ID(func, bpf_lsm_bprm_committed_creds) BTF_ID(func, bpf_lsm_bprm_committing_creds) @@ -298,6 +304,18 @@ BTF_ID(func, bpf_lsm_kernel_module_request) BTF_ID(func, bpf_lsm_kernel_read_file) BTF_ID(func, bpf_lsm_kernfs_init_security) +#ifdef CONFIG_SECURITY_PATH +BTF_ID(func, bpf_lsm_path_unlink) +BTF_ID(func, bpf_lsm_path_mkdir) +BTF_ID(func, bpf_lsm_path_rmdir) +BTF_ID(func, bpf_lsm_path_truncate) +BTF_ID(func, bpf_lsm_path_symlink) +BTF_ID(func, bpf_lsm_path_link) +BTF_ID(func, bpf_lsm_path_rename) +BTF_ID(func, bpf_lsm_path_chmod) +BTF_ID(func, bpf_lsm_path_chown) +#endif /* CONFIG_SECURITY_PATH */ + #ifdef CONFIG_KEYS BTF_ID(func, bpf_lsm_key_free) #endif /* CONFIG_KEYS */ @@ -345,9 +363,8 @@ BTF_ID(func, bpf_lsm_userns_create) BTF_SET_END(sleepable_lsm_hooks) BTF_SET_START(untrusted_lsm_hooks) -BTF_ID(func, bpf_lsm_bpf_map_free_security) -BTF_ID(func, bpf_lsm_bpf_prog_alloc_security) -BTF_ID(func, bpf_lsm_bpf_prog_free_security) +BTF_ID(func, bpf_lsm_bpf_map_free) +BTF_ID(func, bpf_lsm_bpf_prog_free) BTF_ID(func, bpf_lsm_file_alloc_security) BTF_ID(func, bpf_lsm_file_free_security) #ifdef CONFIG_SECURITY_NETWORK diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index db6176fb64dc3f..02068bd0e4d951 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -352,18 +352,24 @@ const struct bpf_link_ops bpf_struct_ops_link_lops = { int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks, struct bpf_tramp_link *link, const struct btf_func_model *model, - void *image, void *image_end) + void *stub_func, void *image, void *image_end) { - u32 flags; + u32 flags = BPF_TRAMP_F_INDIRECT; + int size; tlinks[BPF_TRAMP_FENTRY].links[0] = link; tlinks[BPF_TRAMP_FENTRY].nr_links = 1; - /* BPF_TRAMP_F_RET_FENTRY_RET is only used by bpf_struct_ops, - * and it must be used alone. - */ - flags = model->ret_size > 0 ? BPF_TRAMP_F_RET_FENTRY_RET : 0; + + if (model->ret_size > 0) + flags |= BPF_TRAMP_F_RET_FENTRY_RET; + + size = arch_bpf_trampoline_size(model, flags, tlinks, NULL); + if (size < 0) + return size; + if (size > (unsigned long)image_end - (unsigned long)image) + return -E2BIG; return arch_prepare_bpf_trampoline(NULL, image, image_end, - model, flags, tlinks, NULL); + model, flags, tlinks, stub_func); } static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, @@ -497,11 +503,12 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, err = bpf_struct_ops_prepare_trampoline(tlinks, link, &st_ops->func_models[i], + *(void **)(st_ops->cfi_stubs + moff), image, image_end); if (err < 0) goto reset_unlock; - *(void **)(kdata + moff) = image; + *(void **)(kdata + moff) = image + cfi_get_offset(); image += err; /* put prog_id to udata */ @@ -515,7 +522,7 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, if (err) goto reset_unlock; } - set_memory_rox((long)st_map->image, 1); + arch_protect_bpf_trampoline(st_map->image, PAGE_SIZE); /* Let bpf_link handle registration & unregistration. * * Pair with smp_load_acquire() during lookup_elem(). @@ -524,7 +531,7 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, goto unlock; } - set_memory_rox((long)st_map->image, 1); + arch_protect_bpf_trampoline(st_map->image, PAGE_SIZE); err = st_ops->reg(kdata); if (likely(!err)) { /* This refcnt increment on the map here after @@ -547,8 +554,7 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, * there was a race in registering the struct_ops (under the same name) to * a sub-system through different struct_ops's maps. */ - set_memory_nx((long)st_map->image, 1); - set_memory_rw((long)st_map->image, 1); + arch_unprotect_bpf_trampoline(st_map->image, PAGE_SIZE); reset_unlock: bpf_struct_ops_map_put_progs(st_map); @@ -616,7 +622,7 @@ static void __bpf_struct_ops_map_free(struct bpf_map *map) bpf_struct_ops_map_put_progs(st_map); bpf_map_area_free(st_map->links); if (st_map->image) { - bpf_jit_free_exec(st_map->image); + arch_free_bpf_trampoline(st_map->image, PAGE_SIZE); bpf_jit_uncharge_modmem(PAGE_SIZE); } bpf_map_area_free(st_map->uvalue); @@ -691,7 +697,7 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) return ERR_PTR(ret); } - st_map->image = bpf_jit_alloc_exec(PAGE_SIZE); + st_map->image = arch_alloc_bpf_trampoline(PAGE_SIZE); if (!st_map->image) { /* __bpf_struct_ops_map_free() uses st_map->image as flag * for "charged or not". In this case, we need to unchange @@ -711,7 +717,6 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) } mutex_init(&st_map->lock); - set_vm_flush_reset_perms(st_map->image); bpf_map_init_from_attr(map, attr); return map; diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 63cf4128fc0594..d56433bf8aba1b 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -6956,7 +6956,7 @@ int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog, * (either PTR_TO_CTX or SCALAR_VALUE). */ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, - struct bpf_reg_state *regs, bool is_ex_cb) + struct bpf_reg_state *regs, u32 *arg_cnt) { struct bpf_verifier_log *log = &env->log; struct bpf_prog *prog = env->prog; @@ -7013,6 +7013,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, tname, nargs, MAX_BPF_FUNC_REG_ARGS); return -EINVAL; } + *arg_cnt = nargs; /* check that function returns int, exception cb also requires this */ t = btf_type_by_id(btf, t->type); while (btf_type_is_modifier(t)) @@ -7062,14 +7063,6 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, i, btf_type_str(t), tname); return -EINVAL; } - /* We have already ensured that the callback returns an integer, just - * like all global subprogs. We need to determine it only has a single - * scalar argument. - */ - if (is_ex_cb && (nargs != 1 || regs[BPF_REG_1].type != SCALAR_VALUE)) { - bpf_log(log, "exception cb only supports single integer argument\n"); - return -EINVAL; - } return 0; } diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 491d20038cbe0c..98e0e3835b28b6 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -1630,7 +1630,7 @@ cgroup_dev_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_perf_event_output: return &bpf_event_output_data_proto; default: - return bpf_base_func_proto(func_id); + return bpf_base_func_proto(func_id, prog); } } @@ -2191,7 +2191,7 @@ sysctl_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_perf_event_output: return &bpf_event_output_data_proto; default: - return bpf_base_func_proto(func_id); + return bpf_base_func_proto(func_id, prog); } } @@ -2348,7 +2348,7 @@ cg_sockopt_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_perf_event_output: return &bpf_event_output_data_proto; default: - return bpf_base_func_proto(func_id); + return bpf_base_func_proto(func_id, prog); } } diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index fe254ae035fe49..14ace23d517b7f 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -121,6 +121,9 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag #endif INIT_LIST_HEAD_RCU(&fp->aux->ksym.lnode); +#ifdef CONFIG_FINEIBT + INIT_LIST_HEAD_RCU(&fp->aux->ksym_prefix.lnode); +#endif mutex_init(&fp->aux->used_maps_mutex); mutex_init(&fp->aux->dst_mutex); @@ -679,7 +682,7 @@ static bool bpf_prog_kallsyms_candidate(const struct bpf_prog *fp) void bpf_prog_kallsyms_add(struct bpf_prog *fp) { if (!bpf_prog_kallsyms_candidate(fp) || - !bpf_capable()) + !bpf_token_capable(fp->aux->token, CAP_BPF)) return; bpf_prog_ksym_set_addr(fp); @@ -687,6 +690,23 @@ void bpf_prog_kallsyms_add(struct bpf_prog *fp) fp->aux->ksym.prog = true; bpf_ksym_add(&fp->aux->ksym); + +#ifdef CONFIG_FINEIBT + /* + * When FineIBT, code in the __cfi_foo() symbols can get executed + * and hence unwinder needs help. + */ + if (cfi_mode != CFI_FINEIBT) + return; + + snprintf(fp->aux->ksym_prefix.name, KSYM_NAME_LEN, + "__cfi_%s", fp->aux->ksym.name); + + fp->aux->ksym_prefix.start = (unsigned long) fp->bpf_func - 16; + fp->aux->ksym_prefix.end = (unsigned long) fp->bpf_func; + + bpf_ksym_add(&fp->aux->ksym_prefix); +#endif } void bpf_prog_kallsyms_del(struct bpf_prog *fp) @@ -695,6 +715,11 @@ void bpf_prog_kallsyms_del(struct bpf_prog *fp) return; bpf_ksym_del(&fp->aux->ksym); +#ifdef CONFIG_FINEIBT + if (cfi_mode != CFI_FINEIBT) + return; + bpf_ksym_del(&fp->aux->ksym_prefix); +#endif } static struct bpf_ksym *bpf_ksym_find(unsigned long addr) @@ -932,20 +957,20 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns) return ptr; } -void bpf_prog_pack_free(struct bpf_binary_header *hdr) +void bpf_prog_pack_free(void *ptr, u32 size) { struct bpf_prog_pack *pack = NULL, *tmp; unsigned int nbits; unsigned long pos; mutex_lock(&pack_mutex); - if (hdr->size > BPF_PROG_PACK_SIZE) { - bpf_jit_free_exec(hdr); + if (size > BPF_PROG_PACK_SIZE) { + bpf_jit_free_exec(ptr); goto out; } list_for_each_entry(tmp, &pack_list, list) { - if ((void *)hdr >= tmp->ptr && (tmp->ptr + BPF_PROG_PACK_SIZE) > (void *)hdr) { + if (ptr >= tmp->ptr && (tmp->ptr + BPF_PROG_PACK_SIZE) > ptr) { pack = tmp; break; } @@ -954,10 +979,10 @@ void bpf_prog_pack_free(struct bpf_binary_header *hdr) if (WARN_ONCE(!pack, "bpf_prog_pack bug\n")) goto out; - nbits = BPF_PROG_SIZE_TO_NBITS(hdr->size); - pos = ((unsigned long)hdr - (unsigned long)pack->ptr) >> BPF_PROG_CHUNK_SHIFT; + nbits = BPF_PROG_SIZE_TO_NBITS(size); + pos = ((unsigned long)ptr - (unsigned long)pack->ptr) >> BPF_PROG_CHUNK_SHIFT; - WARN_ONCE(bpf_arch_text_invalidate(hdr, hdr->size), + WARN_ONCE(bpf_arch_text_invalidate(ptr, size), "bpf_prog_pack bug: missing bpf_arch_text_invalidate?\n"); bitmap_clear(pack->bitmap, pos, nbits); @@ -1104,8 +1129,7 @@ bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **image_ptr, *rw_header = kvmalloc(size, GFP_KERNEL); if (!*rw_header) { - bpf_arch_text_copy(&ro_header->size, &size, sizeof(size)); - bpf_prog_pack_free(ro_header); + bpf_prog_pack_free(ro_header, size); bpf_jit_uncharge_modmem(size); return NULL; } @@ -1136,7 +1160,7 @@ int bpf_jit_binary_pack_finalize(struct bpf_prog *prog, kvfree(rw_header); if (IS_ERR(ptr)) { - bpf_prog_pack_free(ro_header); + bpf_prog_pack_free(ro_header, ro_header->size); return PTR_ERR(ptr); } return 0; @@ -1157,7 +1181,7 @@ void bpf_jit_binary_pack_free(struct bpf_binary_header *ro_header, { u32 size = ro_header->size; - bpf_prog_pack_free(ro_header); + bpf_prog_pack_free(ro_header, size); kvfree(rw_header); bpf_jit_uncharge_modmem(size); } @@ -2668,12 +2692,16 @@ void __bpf_free_used_maps(struct bpf_prog_aux *aux, struct bpf_map **used_maps, u32 len) { struct bpf_map *map; + bool sleepable; u32 i; + sleepable = aux->sleepable; for (i = 0; i < len; i++) { map = used_maps[i]; if (map->ops->map_poke_untrack) map->ops->map_poke_untrack(map, aux); + if (sleepable) + atomic64_dec(&map->sleepable_refcnt); bpf_map_put(map); } } @@ -2751,6 +2779,7 @@ void bpf_prog_free(struct bpf_prog *fp) if (aux->dst_prog) bpf_prog_put(aux->dst_prog); + bpf_token_put(aux->token); INIT_WORK(&aux->work, bpf_prog_free_deferred); schedule_work(&aux->work); } diff --git a/kernel/bpf/cpumask.c b/kernel/bpf/cpumask.c index e01c741e54e7b8..2e73533a3811cd 100644 --- a/kernel/bpf/cpumask.c +++ b/kernel/bpf/cpumask.c @@ -96,6 +96,12 @@ __bpf_kfunc void bpf_cpumask_release(struct bpf_cpumask *cpumask) migrate_enable(); } +__bpf_kfunc void bpf_cpumask_release_dtor(void *cpumask) +{ + bpf_cpumask_release(cpumask); +} +CFI_NOSEAL(bpf_cpumask_release_dtor); + /** * bpf_cpumask_first() - Get the index of the first nonzero bit in the cpumask. * @cpumask: The cpumask being queried. @@ -405,6 +411,17 @@ __bpf_kfunc u32 bpf_cpumask_any_and_distribute(const struct cpumask *src1, return cpumask_any_and_distribute(src1, src2); } +/** + * bpf_cpumask_weight() - Return the number of bits in @cpumask. + * @cpumask: The cpumask being queried. + * + * Count the number of set bits in the given cpumask. + */ +__bpf_kfunc u32 bpf_cpumask_weight(const struct cpumask *cpumask) +{ + return cpumask_weight(cpumask); +} + __bpf_kfunc_end_defs(); BTF_SET8_START(cpumask_kfunc_btf_ids) @@ -432,6 +449,7 @@ BTF_ID_FLAGS(func, bpf_cpumask_full, KF_RCU) BTF_ID_FLAGS(func, bpf_cpumask_copy, KF_RCU) BTF_ID_FLAGS(func, bpf_cpumask_any_distribute, KF_RCU) BTF_ID_FLAGS(func, bpf_cpumask_any_and_distribute, KF_RCU) +BTF_ID_FLAGS(func, bpf_cpumask_weight, KF_RCU) BTF_SET8_END(cpumask_kfunc_btf_ids) static const struct btf_kfunc_id_set cpumask_kfunc_set = { @@ -441,7 +459,7 @@ static const struct btf_kfunc_id_set cpumask_kfunc_set = { BTF_ID_LIST(cpumask_dtor_ids) BTF_ID(struct, bpf_cpumask) -BTF_ID(func, bpf_cpumask_release) +BTF_ID(func, bpf_cpumask_release_dtor) static int __init cpumask_kfunc_init(void) { diff --git a/kernel/bpf/dispatcher.c b/kernel/bpf/dispatcher.c index fa3e9225aedc0a..70fb82bf16370e 100644 --- a/kernel/bpf/dispatcher.c +++ b/kernel/bpf/dispatcher.c @@ -150,14 +150,11 @@ void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, goto out; d->rw_image = bpf_jit_alloc_exec(PAGE_SIZE); if (!d->rw_image) { - u32 size = PAGE_SIZE; - - bpf_arch_text_copy(d->image, &size, sizeof(size)); - bpf_prog_pack_free((struct bpf_binary_header *)d->image); + bpf_prog_pack_free(d->image, PAGE_SIZE); d->image = NULL; goto out; } - bpf_image_ksym_add(d->image, &d->ksym); + bpf_image_ksym_add(d->image, PAGE_SIZE, &d->ksym); } prev_num_progs = d->num_progs; diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index fd8d4b0addfca0..ec3bdcc6a3cfd0 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -897,7 +897,7 @@ static void htab_put_fd_value(struct bpf_htab *htab, struct htab_elem *l) if (map->ops->map_fd_put_ptr) { ptr = fd_htab_map_get_ptr(map, l); - map->ops->map_fd_put_ptr(ptr); + map->ops->map_fd_put_ptr(map, ptr, true); } } @@ -2484,7 +2484,7 @@ static void fd_htab_map_free(struct bpf_map *map) hlist_nulls_for_each_entry_safe(l, n, head, hash_node) { void *ptr = fd_htab_map_get_ptr(map, l); - map->ops->map_fd_put_ptr(ptr); + map->ops->map_fd_put_ptr(map, ptr, false); } } @@ -2523,9 +2523,15 @@ int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file, if (IS_ERR(ptr)) return PTR_ERR(ptr); + /* The htab bucket lock is always held during update operations in fd + * htab map, and the following rcu_read_lock() is only used to avoid + * the WARN_ON_ONCE in htab_map_update_elem(). + */ + rcu_read_lock(); ret = htab_map_update_elem(map, key, &ptr, map_flags); + rcu_read_unlock(); if (ret) - map->ops->map_fd_put_ptr(ptr); + map->ops->map_fd_put_ptr(map, ptr, false); return ret; } diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index b45a8381f9bda7..07fd4b5704f368 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -32,12 +32,13 @@ * * Different map implementations will rely on rcu in map methods * lookup/update/delete, therefore eBPF programs must run under rcu lock - * if program is allowed to access maps, so check rcu_read_lock_held in - * all three functions. + * if program is allowed to access maps, so check rcu_read_lock_held() or + * rcu_read_lock_trace_held() in all three functions. */ BPF_CALL_2(bpf_map_lookup_elem, struct bpf_map *, map, void *, key) { - WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held()); + WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() && + !rcu_read_lock_bh_held()); return (unsigned long) map->ops->map_lookup_elem(map, key); } @@ -53,7 +54,8 @@ const struct bpf_func_proto bpf_map_lookup_elem_proto = { BPF_CALL_4(bpf_map_update_elem, struct bpf_map *, map, void *, key, void *, value, u64, flags) { - WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held()); + WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() && + !rcu_read_lock_bh_held()); return map->ops->map_update_elem(map, key, value, flags); } @@ -70,7 +72,8 @@ const struct bpf_func_proto bpf_map_update_elem_proto = { BPF_CALL_2(bpf_map_delete_elem, struct bpf_map *, map, void *, key) { - WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held()); + WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() && + !rcu_read_lock_bh_held()); return map->ops->map_delete_elem(map, key); } @@ -1676,7 +1679,7 @@ const struct bpf_func_proto bpf_probe_read_kernel_str_proto __weak; const struct bpf_func_proto bpf_task_pt_regs_proto __weak; const struct bpf_func_proto * -bpf_base_func_proto(enum bpf_func_id func_id) +bpf_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { switch (func_id) { case BPF_FUNC_map_lookup_elem: @@ -1727,7 +1730,7 @@ bpf_base_func_proto(enum bpf_func_id func_id) break; } - if (!bpf_capable()) + if (!bpf_token_capable(prog->aux->token, CAP_BPF)) return NULL; switch (func_id) { @@ -1785,7 +1788,7 @@ bpf_base_func_proto(enum bpf_func_id func_id) break; } - if (!perfmon_capable()) + if (!bpf_token_capable(prog->aux->token, CAP_PERFMON)) return NULL; switch (func_id) { @@ -2147,6 +2150,12 @@ __bpf_kfunc void bpf_task_release(struct task_struct *p) put_task_struct_rcu_user(p); } +__bpf_kfunc void bpf_task_release_dtor(void *p) +{ + put_task_struct_rcu_user(p); +} +CFI_NOSEAL(bpf_task_release_dtor); + #ifdef CONFIG_CGROUPS /** * bpf_cgroup_acquire - Acquire a reference to a cgroup. A cgroup acquired by @@ -2171,6 +2180,12 @@ __bpf_kfunc void bpf_cgroup_release(struct cgroup *cgrp) cgroup_put(cgrp); } +__bpf_kfunc void bpf_cgroup_release_dtor(void *cgrp) +{ + cgroup_put(cgrp); +} +CFI_NOSEAL(bpf_cgroup_release_dtor); + /** * bpf_cgroup_ancestor - Perform a lookup on an entry in a cgroup's ancestor * array. A cgroup returned by this kfunc which is not subsequently stored in a @@ -2522,7 +2537,7 @@ __bpf_kfunc void bpf_throw(u64 cookie) * which skips compiler generated instrumentation to do the same. */ kasan_unpoison_task_stack_below((void *)(long)ctx.sp); - ctx.aux->bpf_exception_cb(cookie, ctx.sp, ctx.bp); + ctx.aux->bpf_exception_cb(cookie, ctx.sp, ctx.bp, 0, 0); WARN(1, "A call to BPF exception callback should never return\n"); } @@ -2567,10 +2582,10 @@ static const struct btf_kfunc_id_set generic_kfunc_set = { BTF_ID_LIST(generic_dtor_ids) BTF_ID(struct, task_struct) -BTF_ID(func, bpf_task_release) +BTF_ID(func, bpf_task_release_dtor) #ifdef CONFIG_CGROUPS BTF_ID(struct, cgroup) -BTF_ID(func, bpf_cgroup_release) +BTF_ID(func, bpf_cgroup_release_dtor) #endif BTF_SET8_START(common_btf_ids) @@ -2627,6 +2642,7 @@ static int __init kfunc_init(void) ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &generic_kfunc_set); ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &generic_kfunc_set); + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_XDP, &generic_kfunc_set); ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &generic_kfunc_set); ret = ret ?: register_btf_id_dtor_kfuncs(generic_dtors, ARRAY_SIZE(generic_dtors), diff --git a/kernel/bpf/inode.c b/kernel/bpf/inode.c index 1aafb2ff2e953c..4383b3d13a557a 100644 --- a/kernel/bpf/inode.c +++ b/kernel/bpf/inode.c @@ -20,6 +20,7 @@ #include #include #include +#include #include "preload/bpf_preload.h" enum bpf_type { @@ -98,9 +99,9 @@ static const struct inode_operations bpf_prog_iops = { }; static const struct inode_operations bpf_map_iops = { }; static const struct inode_operations bpf_link_iops = { }; -static struct inode *bpf_get_inode(struct super_block *sb, - const struct inode *dir, - umode_t mode) +struct inode *bpf_get_inode(struct super_block *sb, + const struct inode *dir, + umode_t mode) { struct inode *inode; @@ -594,15 +595,183 @@ struct bpf_prog *bpf_prog_get_type_path(const char *name, enum bpf_prog_type typ } EXPORT_SYMBOL(bpf_prog_get_type_path); +struct bpffs_btf_enums { + const struct btf *btf; + const struct btf_type *cmd_t; + const struct btf_type *map_t; + const struct btf_type *prog_t; + const struct btf_type *attach_t; +}; + +static int find_bpffs_btf_enums(struct bpffs_btf_enums *info) +{ + const struct btf *btf; + const struct btf_type *t; + const char *name; + int i, n; + + memset(info, 0, sizeof(*info)); + + btf = bpf_get_btf_vmlinux(); + if (IS_ERR(btf)) + return PTR_ERR(btf); + if (!btf) + return -ENOENT; + + info->btf = btf; + + for (i = 1, n = btf_nr_types(btf); i < n; i++) { + t = btf_type_by_id(btf, i); + if (!btf_type_is_enum(t)) + continue; + + name = btf_name_by_offset(btf, t->name_off); + if (!name) + continue; + + if (strcmp(name, "bpf_cmd") == 0) + info->cmd_t = t; + else if (strcmp(name, "bpf_map_type") == 0) + info->map_t = t; + else if (strcmp(name, "bpf_prog_type") == 0) + info->prog_t = t; + else if (strcmp(name, "bpf_attach_type") == 0) + info->attach_t = t; + else + continue; + + if (info->cmd_t && info->map_t && info->prog_t && info->attach_t) + return 0; + } + + return -ESRCH; +} + +static bool find_btf_enum_const(const struct btf *btf, const struct btf_type *enum_t, + const char *prefix, const char *str, int *value) +{ + const struct btf_enum *e; + const char *name; + int i, n, pfx_len = strlen(prefix); + + *value = 0; + + if (!btf || !enum_t) + return false; + + for (i = 0, n = btf_vlen(enum_t); i < n; i++) { + e = &btf_enum(enum_t)[i]; + + name = btf_name_by_offset(btf, e->name_off); + if (!name || strncasecmp(name, prefix, pfx_len) != 0) + continue; + + /* match symbolic name case insensitive and ignoring prefix */ + if (strcasecmp(name + pfx_len, str) == 0) { + *value = e->val; + return true; + } + } + + return false; +} + +static void seq_print_delegate_opts(struct seq_file *m, + const char *opt_name, + const struct btf *btf, + const struct btf_type *enum_t, + const char *prefix, + u64 delegate_msk, u64 any_msk) +{ + const struct btf_enum *e; + bool first = true; + const char *name; + u64 msk; + int i, n, pfx_len = strlen(prefix); + + delegate_msk &= any_msk; /* clear unknown bits */ + + if (delegate_msk == 0) + return; + + seq_printf(m, ",%s", opt_name); + if (delegate_msk == any_msk) { + seq_printf(m, "=any"); + return; + } + + if (btf && enum_t) { + for (i = 0, n = btf_vlen(enum_t); i < n; i++) { + e = &btf_enum(enum_t)[i]; + name = btf_name_by_offset(btf, e->name_off); + if (!name || strncasecmp(name, prefix, pfx_len) != 0) + continue; + msk = 1ULL << e->val; + if (delegate_msk & msk) { + /* emit lower-case name without prefix */ + seq_printf(m, "%c", first ? '=' : ':'); + name += pfx_len; + while (*name) { + seq_printf(m, "%c", tolower(*name)); + name++; + } + + delegate_msk &= ~msk; + first = false; + } + } + } + if (delegate_msk) + seq_printf(m, "%c0x%llx", first ? '=' : ':', delegate_msk); +} + /* * Display the mount options in /proc/mounts. */ static int bpf_show_options(struct seq_file *m, struct dentry *root) { - umode_t mode = d_inode(root)->i_mode & S_IALLUGO & ~S_ISVTX; - + struct bpf_mount_opts *opts = root->d_sb->s_fs_info; + struct inode *inode = d_inode(root); + umode_t mode = inode->i_mode & S_IALLUGO & ~S_ISVTX; + u64 mask; + + if (!uid_eq(inode->i_uid, GLOBAL_ROOT_UID)) + seq_printf(m, ",uid=%u", + from_kuid_munged(&init_user_ns, inode->i_uid)); + if (!gid_eq(inode->i_gid, GLOBAL_ROOT_GID)) + seq_printf(m, ",gid=%u", + from_kgid_munged(&init_user_ns, inode->i_gid)); if (mode != S_IRWXUGO) seq_printf(m, ",mode=%o", mode); + + if (opts->delegate_cmds || opts->delegate_maps || + opts->delegate_progs || opts->delegate_attachs) { + struct bpffs_btf_enums info; + + /* ignore errors, fallback to hex */ + (void)find_bpffs_btf_enums(&info); + + mask = (1ULL << __MAX_BPF_CMD) - 1; + seq_print_delegate_opts(m, "delegate_cmds", + info.btf, info.cmd_t, "BPF_", + opts->delegate_cmds, mask); + + mask = (1ULL << __MAX_BPF_MAP_TYPE) - 1; + seq_print_delegate_opts(m, "delegate_maps", + info.btf, info.map_t, "BPF_MAP_TYPE_", + opts->delegate_maps, mask); + + mask = (1ULL << __MAX_BPF_PROG_TYPE) - 1; + seq_print_delegate_opts(m, "delegate_progs", + info.btf, info.prog_t, "BPF_PROG_TYPE_", + opts->delegate_progs, mask); + + mask = (1ULL << __MAX_BPF_ATTACH_TYPE) - 1; + seq_print_delegate_opts(m, "delegate_attachs", + info.btf, info.attach_t, "BPF_", + opts->delegate_attachs, mask); + } + return 0; } @@ -617,7 +786,7 @@ static void bpf_free_inode(struct inode *inode) free_inode_nonrcu(inode); } -static const struct super_operations bpf_super_ops = { +const struct super_operations bpf_super_ops = { .statfs = simple_statfs, .drop_inode = generic_delete_inode, .show_options = bpf_show_options, @@ -625,23 +794,33 @@ static const struct super_operations bpf_super_ops = { }; enum { + OPT_UID, + OPT_GID, OPT_MODE, + OPT_DELEGATE_CMDS, + OPT_DELEGATE_MAPS, + OPT_DELEGATE_PROGS, + OPT_DELEGATE_ATTACHS, }; static const struct fs_parameter_spec bpf_fs_parameters[] = { + fsparam_u32 ("uid", OPT_UID), + fsparam_u32 ("gid", OPT_GID), fsparam_u32oct ("mode", OPT_MODE), + fsparam_string ("delegate_cmds", OPT_DELEGATE_CMDS), + fsparam_string ("delegate_maps", OPT_DELEGATE_MAPS), + fsparam_string ("delegate_progs", OPT_DELEGATE_PROGS), + fsparam_string ("delegate_attachs", OPT_DELEGATE_ATTACHS), {} }; -struct bpf_mount_opts { - umode_t mode; -}; - static int bpf_parse_param(struct fs_context *fc, struct fs_parameter *param) { - struct bpf_mount_opts *opts = fc->fs_private; + struct bpf_mount_opts *opts = fc->s_fs_info; struct fs_parse_result result; - int opt; + kuid_t uid; + kgid_t gid; + int opt, err; opt = fs_parse(fc, bpf_fs_parameters, param, &result); if (opt < 0) { @@ -662,12 +841,104 @@ static int bpf_parse_param(struct fs_context *fc, struct fs_parameter *param) } switch (opt) { + case OPT_UID: + uid = make_kuid(current_user_ns(), result.uint_32); + if (!uid_valid(uid)) + goto bad_value; + + /* + * The requested uid must be representable in the + * filesystem's idmapping. + */ + if (!kuid_has_mapping(fc->user_ns, uid)) + goto bad_value; + + opts->uid = uid; + break; + case OPT_GID: + gid = make_kgid(current_user_ns(), result.uint_32); + if (!gid_valid(gid)) + goto bad_value; + + /* + * The requested gid must be representable in the + * filesystem's idmapping. + */ + if (!kgid_has_mapping(fc->user_ns, gid)) + goto bad_value; + + opts->gid = gid; + break; case OPT_MODE: opts->mode = result.uint_32 & S_IALLUGO; break; + case OPT_DELEGATE_CMDS: + case OPT_DELEGATE_MAPS: + case OPT_DELEGATE_PROGS: + case OPT_DELEGATE_ATTACHS: { + struct bpffs_btf_enums info; + const struct btf_type *enum_t; + const char *enum_pfx; + u64 *delegate_msk, msk = 0; + char *p; + int val; + + /* ignore errors, fallback to hex */ + (void)find_bpffs_btf_enums(&info); + + switch (opt) { + case OPT_DELEGATE_CMDS: + delegate_msk = &opts->delegate_cmds; + enum_t = info.cmd_t; + enum_pfx = "BPF_"; + break; + case OPT_DELEGATE_MAPS: + delegate_msk = &opts->delegate_maps; + enum_t = info.map_t; + enum_pfx = "BPF_MAP_TYPE_"; + break; + case OPT_DELEGATE_PROGS: + delegate_msk = &opts->delegate_progs; + enum_t = info.prog_t; + enum_pfx = "BPF_PROG_TYPE_"; + break; + case OPT_DELEGATE_ATTACHS: + delegate_msk = &opts->delegate_attachs; + enum_t = info.attach_t; + enum_pfx = "BPF_"; + break; + default: + return -EINVAL; + } + + while ((p = strsep(¶m->string, ":"))) { + if (strcmp(p, "any") == 0) { + msk |= ~0ULL; + } else if (find_btf_enum_const(info.btf, enum_t, enum_pfx, p, &val)) { + msk |= 1ULL << val; + } else { + err = kstrtou64(p, 0, &msk); + if (err) + return err; + } + } + + /* Setting delegation mount options requires privileges */ + if (msk && !capable(CAP_SYS_ADMIN)) + return -EPERM; + + *delegate_msk |= msk; + break; + } + default: + /* ignore unknown mount options */ + break; } return 0; + +bad_value: + return invalfc(fc, "Bad value for '%s'", param->key); } struct bpf_preload_ops *bpf_preload_ops; @@ -739,10 +1010,14 @@ static int populate_bpffs(struct dentry *parent) static int bpf_fill_super(struct super_block *sb, struct fs_context *fc) { static const struct tree_descr bpf_rfiles[] = { { "" } }; - struct bpf_mount_opts *opts = fc->fs_private; + struct bpf_mount_opts *opts = sb->s_fs_info; struct inode *inode; int ret; + /* Mounting an instance of BPF FS requires privileges */ + if (fc->user_ns != &init_user_ns && !capable(CAP_SYS_ADMIN)) + return -EPERM; + ret = simple_fill_super(sb, BPF_FS_MAGIC, bpf_rfiles); if (ret) return ret; @@ -750,6 +1025,8 @@ static int bpf_fill_super(struct super_block *sb, struct fs_context *fc) sb->s_op = &bpf_super_ops; inode = sb->s_root->d_inode; + inode->i_uid = opts->uid; + inode->i_gid = opts->gid; inode->i_op = &bpf_dir_iops; inode->i_mode &= ~S_IALLUGO; populate_bpffs(sb->s_root); @@ -764,7 +1041,7 @@ static int bpf_get_tree(struct fs_context *fc) static void bpf_free_fc(struct fs_context *fc) { - kfree(fc->fs_private); + kfree(fc->s_fs_info); } static const struct fs_context_operations bpf_context_ops = { @@ -785,18 +1062,35 @@ static int bpf_init_fs_context(struct fs_context *fc) return -ENOMEM; opts->mode = S_IRWXUGO; + opts->uid = current_fsuid(); + opts->gid = current_fsgid(); + + /* start out with no BPF token delegation enabled */ + opts->delegate_cmds = 0; + opts->delegate_maps = 0; + opts->delegate_progs = 0; + opts->delegate_attachs = 0; - fc->fs_private = opts; + fc->s_fs_info = opts; fc->ops = &bpf_context_ops; return 0; } +static void bpf_kill_super(struct super_block *sb) +{ + struct bpf_mount_opts *opts = sb->s_fs_info; + + kill_litter_super(sb); + kfree(opts); +} + static struct file_system_type bpf_fs_type = { .owner = THIS_MODULE, .name = "bpf", .init_fs_context = bpf_init_fs_context, .parameters = bpf_fs_parameters, - .kill_sb = kill_litter_super, + .kill_sb = bpf_kill_super, + .fs_flags = FS_USERNS_MOUNT, }; static int __init bpf_init(void) diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c index 3505f3e5ae9630..594a234f122bbb 100644 --- a/kernel/bpf/log.c +++ b/kernel/bpf/log.c @@ -539,6 +539,19 @@ static void verbose_snum(struct bpf_verifier_env *env, s64 num) verbose(env, "%#llx", num); } +int tnum_strn(char *str, size_t size, struct tnum a) +{ + /* print as a constant, if tnum is fully known */ + if (a.mask == 0) { + if (is_unum_decimal(a.value)) + return snprintf(str, size, "%llu", a.value); + else + return snprintf(str, size, "%#llx", a.value); + } + return snprintf(str, size, "(%#llx; %#llx)", a.value, a.mask); +} +EXPORT_SYMBOL_GPL(tnum_strn); + static void print_scalar_ranges(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, const char **sep) @@ -615,6 +628,12 @@ static bool type_is_map_ptr(enum bpf_reg_type t) { } } +/* + * _a stands for append, was shortened to avoid multiline statements below. + * This macro is used to output a comma separated list of attributes. + */ +#define verbose_a(fmt, ...) ({ verbose(env, "%s" fmt, sep, ##__VA_ARGS__); sep = ","; }) + static void print_reg_state(struct bpf_verifier_env *env, const struct bpf_func_state *state, const struct bpf_reg_state *reg) @@ -630,11 +649,6 @@ static void print_reg_state(struct bpf_verifier_env *env, verbose_snum(env, reg->var_off.value + reg->off); return; } -/* - * _a stands for append, was shortened to avoid multiline statements below. - * This macro is used to output a comma separated list of attributes. - */ -#define verbose_a(fmt, ...) ({ verbose(env, "%s" fmt, sep, ##__VA_ARGS__); sep = ","; }) verbose(env, "%s", reg_type_str(env, t)); if (t == PTR_TO_STACK) { @@ -669,6 +683,12 @@ static void print_reg_state(struct bpf_verifier_env *env, verbose_a("r="); verbose_unum(env, reg->range); } + if (base_type(t) == PTR_TO_MEM) { + verbose_a("sz="); + verbose_unum(env, reg->mem_size); + } + if (t == CONST_PTR_TO_DYNPTR) + verbose_a("type=%s", dynptr_type_str(reg->dynptr.type)); if (tnum_is_const(reg->var_off)) { /* a pointer register with fixed offset */ if (reg->var_off.value) { @@ -685,8 +705,6 @@ static void print_reg_state(struct bpf_verifier_env *env, } } verbose(env, ")"); - -#undef verbose_a } void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_func_state *state, @@ -710,6 +728,7 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_func_st } for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { char types_buf[BPF_REG_SIZE + 1]; + const char *sep = ""; bool valid = false; u8 slot_type; int j; @@ -748,9 +767,14 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_func_st verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); print_liveness(env, reg->live); - verbose(env, "=dynptr_%s", dynptr_type_str(reg->dynptr.type)); + verbose(env, "=dynptr_%s(", dynptr_type_str(reg->dynptr.type)); + if (reg->id) + verbose_a("id=%d", reg->id); if (reg->ref_obj_id) - verbose(env, "(ref_id=%d)", reg->ref_obj_id); + verbose_a("ref_id=%d", reg->ref_obj_id); + if (reg->dynptr_id) + verbose_a("dynptr_id=%d", reg->dynptr_id); + verbose(env, ")"); break; case STACK_ITER: /* only main slot has ref_obj_id set; skip others */ diff --git a/kernel/bpf/map_in_map.c b/kernel/bpf/map_in_map.c index cd5eafaba97e22..8ef269e66ba502 100644 --- a/kernel/bpf/map_in_map.c +++ b/kernel/bpf/map_in_map.c @@ -127,12 +127,21 @@ void *bpf_map_fd_get_ptr(struct bpf_map *map, return inner_map; } -void bpf_map_fd_put_ptr(void *ptr) +void bpf_map_fd_put_ptr(struct bpf_map *map, void *ptr, bool need_defer) { - /* ptr->ops->map_free() has to go through one - * rcu grace period by itself. + struct bpf_map *inner_map = ptr; + + /* Defer the freeing of inner map according to the sleepable attribute + * of bpf program which owns the outer map, so unnecessary waiting for + * RCU tasks trace grace period can be avoided. */ - bpf_map_put(ptr); + if (need_defer) { + if (atomic64_read(&map->sleepable_refcnt)) + WRITE_ONCE(inner_map->free_after_mult_rcu_gp, true); + else + WRITE_ONCE(inner_map->free_after_rcu_gp, true); + } + bpf_map_put(inner_map); } u32 bpf_map_fd_sys_lookup_elem(void *ptr) diff --git a/kernel/bpf/map_in_map.h b/kernel/bpf/map_in_map.h index bcb7534afb3c0d..7d61602354de80 100644 --- a/kernel/bpf/map_in_map.h +++ b/kernel/bpf/map_in_map.h @@ -13,7 +13,7 @@ struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd); void bpf_map_meta_free(struct bpf_map *map_meta); void *bpf_map_fd_get_ptr(struct bpf_map *map, struct file *map_file, int ufd); -void bpf_map_fd_put_ptr(void *ptr); +void bpf_map_fd_put_ptr(struct bpf_map *map, void *ptr, bool need_defer); u32 bpf_map_fd_sys_lookup_elem(void *ptr); #endif diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 5e43ddd1b83f6e..8faa1a20edf8f8 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -142,9 +142,13 @@ static u32 bpf_map_value_size(const struct bpf_map *map) static void maybe_wait_bpf_programs(struct bpf_map *map) { - /* Wait for any running BPF programs to complete so that - * userspace, when we return to it, knows that all programs - * that could be running use the new map value. + /* Wait for any running non-sleepable BPF programs to complete so that + * userspace, when we return to it, knows that all non-sleepable + * programs that could be running use the new map value. For sleepable + * BPF programs, synchronize_rcu_tasks_trace() should be used to wait + * for the completions of these programs, but considering the waiting + * time can be very long and userspace may think it will hang forever, + * so don't handle sleepable BPF programs now. */ if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS || map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) @@ -180,15 +184,11 @@ static int bpf_map_update_value(struct bpf_map *map, struct file *map_file, err = bpf_percpu_cgroup_storage_update(map, key, value, flags); } else if (IS_FD_ARRAY(map)) { - rcu_read_lock(); err = bpf_fd_array_map_update_elem(map, map_file, key, value, flags); - rcu_read_unlock(); } else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) { - rcu_read_lock(); err = bpf_fd_htab_map_update_elem(map, map_file, key, value, flags); - rcu_read_unlock(); } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) { /* rcu_read_lock() is not needed */ err = bpf_fd_reuseport_array_update_elem(map, key, value, @@ -203,7 +203,6 @@ static int bpf_map_update_value(struct bpf_map *map, struct file *map_file, rcu_read_unlock(); } bpf_enable_instrumentation(); - maybe_wait_bpf_programs(map); return err; } @@ -264,7 +263,6 @@ static int bpf_map_copy_value(struct bpf_map *map, void *key, void *value, } bpf_enable_instrumentation(); - maybe_wait_bpf_programs(map); return err; } @@ -694,6 +692,7 @@ static void bpf_map_free_deferred(struct work_struct *work) { struct bpf_map *map = container_of(work, struct bpf_map, work); struct btf_record *rec = map->record; + struct btf *btf = map->btf; security_bpf_map_free(map); bpf_map_release_memcg(map); @@ -709,6 +708,10 @@ static void bpf_map_free_deferred(struct work_struct *work) * template bpf_map struct used during verification. */ btf_record_free(rec); + /* Delay freeing of btf for maps, as map_free callback may need + * struct_meta info which will be freed with btf_put(). + */ + btf_put(btf); } static void bpf_map_put_uref(struct bpf_map *map) @@ -719,6 +722,28 @@ static void bpf_map_put_uref(struct bpf_map *map) } } +static void bpf_map_free_in_work(struct bpf_map *map) +{ + INIT_WORK(&map->work, bpf_map_free_deferred); + /* Avoid spawning kworkers, since they all might contend + * for the same mutex like slab_mutex. + */ + queue_work(system_unbound_wq, &map->work); +} + +static void bpf_map_free_rcu_gp(struct rcu_head *rcu) +{ + bpf_map_free_in_work(container_of(rcu, struct bpf_map, rcu)); +} + +static void bpf_map_free_mult_rcu_gp(struct rcu_head *rcu) +{ + if (rcu_trace_implies_rcu_gp()) + bpf_map_free_rcu_gp(rcu); + else + call_rcu(rcu, bpf_map_free_rcu_gp); +} + /* decrement map refcnt and schedule it for freeing via workqueue * (underlying map implementation ops->map_free() might sleep) */ @@ -727,12 +752,14 @@ void bpf_map_put(struct bpf_map *map) if (atomic64_dec_and_test(&map->refcnt)) { /* bpf_map_free_id() must be called first */ bpf_map_free_id(map); - btf_put(map->btf); - INIT_WORK(&map->work, bpf_map_free_deferred); - /* Avoid spawning kworkers, since they all might contend - * for the same mutex like slab_mutex. - */ - queue_work(system_unbound_wq, &map->work); + + WARN_ON_ONCE(atomic64_read(&map->sleepable_refcnt)); + if (READ_ONCE(map->free_after_mult_rcu_gp)) + call_rcu_tasks_trace(&map->rcu, bpf_map_free_mult_rcu_gp); + else if (READ_ONCE(map->free_after_rcu_gp)) + call_rcu(&map->rcu, bpf_map_free_rcu_gp); + else + bpf_map_free_in_work(map); } } EXPORT_SYMBOL_GPL(bpf_map_put); @@ -984,8 +1011,8 @@ int map_check_no_btf(const struct bpf_map *map, return -ENOTSUPP; } -static int map_check_btf(struct bpf_map *map, const struct btf *btf, - u32 btf_key_id, u32 btf_value_id) +static int map_check_btf(struct bpf_map *map, struct bpf_token *token, + const struct btf *btf, u32 btf_key_id, u32 btf_value_id) { const struct btf_type *key_type, *value_type; u32 key_size, value_size; @@ -1013,7 +1040,7 @@ static int map_check_btf(struct bpf_map *map, const struct btf *btf, if (!IS_ERR_OR_NULL(map->record)) { int i; - if (!bpf_capable()) { + if (!bpf_token_capable(token, CAP_BPF)) { ret = -EPERM; goto free_map_tab; } @@ -1096,11 +1123,17 @@ static int map_check_btf(struct bpf_map *map, const struct btf *btf, return ret; } -#define BPF_MAP_CREATE_LAST_FIELD map_extra +static bool bpf_net_capable(void) +{ + return capable(CAP_NET_ADMIN) || capable(CAP_SYS_ADMIN); +} + +#define BPF_MAP_CREATE_LAST_FIELD map_token_fd /* called via syscall */ static int map_create(union bpf_attr *attr) { const struct bpf_map_ops *ops; + struct bpf_token *token = NULL; int numa_node = bpf_map_attr_numa_node(attr); u32 map_type = attr->map_type; struct bpf_map *map; @@ -1151,14 +1184,32 @@ static int map_create(union bpf_attr *attr) if (!ops->map_mem_usage) return -EINVAL; + if (attr->map_token_fd) { + token = bpf_token_get_from_fd(attr->map_token_fd); + if (IS_ERR(token)) + return PTR_ERR(token); + + /* if current token doesn't grant map creation permissions, + * then we can't use this token, so ignore it and rely on + * system-wide capabilities checks + */ + if (!bpf_token_allow_cmd(token, BPF_MAP_CREATE) || + !bpf_token_allow_map_type(token, attr->map_type)) { + bpf_token_put(token); + token = NULL; + } + } + + err = -EPERM; + /* Intent here is for unprivileged_bpf_disabled to block BPF map * creation for unprivileged users; other actions depend * on fd availability and access to bpffs, so are dependent on * object creation success. Even with unprivileged BPF disabled, * capability checks are still carried out. */ - if (sysctl_unprivileged_bpf_disabled && !bpf_capable()) - return -EPERM; + if (sysctl_unprivileged_bpf_disabled && !bpf_token_capable(token, CAP_BPF)) + goto put_token; /* check privileged map type permissions */ switch (map_type) { @@ -1191,25 +1242,27 @@ static int map_create(union bpf_attr *attr) case BPF_MAP_TYPE_LRU_PERCPU_HASH: case BPF_MAP_TYPE_STRUCT_OPS: case BPF_MAP_TYPE_CPUMAP: - if (!bpf_capable()) - return -EPERM; + if (!bpf_token_capable(token, CAP_BPF)) + goto put_token; break; case BPF_MAP_TYPE_SOCKMAP: case BPF_MAP_TYPE_SOCKHASH: case BPF_MAP_TYPE_DEVMAP: case BPF_MAP_TYPE_DEVMAP_HASH: case BPF_MAP_TYPE_XSKMAP: - if (!capable(CAP_NET_ADMIN)) - return -EPERM; + if (!bpf_token_capable(token, CAP_NET_ADMIN)) + goto put_token; break; default: WARN(1, "unsupported map type %d", map_type); - return -EPERM; + goto put_token; } map = ops->map_alloc(attr); - if (IS_ERR(map)) - return PTR_ERR(map); + if (IS_ERR(map)) { + err = PTR_ERR(map); + goto put_token; + } map->ops = ops; map->map_type = map_type; @@ -1246,7 +1299,7 @@ static int map_create(union bpf_attr *attr) map->btf = btf; if (attr->btf_value_type_id) { - err = map_check_btf(map, btf, attr->btf_key_type_id, + err = map_check_btf(map, token, btf, attr->btf_key_type_id, attr->btf_value_type_id); if (err) goto free_map; @@ -1258,15 +1311,16 @@ static int map_create(union bpf_attr *attr) attr->btf_vmlinux_value_type_id; } - err = security_bpf_map_alloc(map); + err = security_bpf_map_create(map, attr, token); if (err) - goto free_map; + goto free_map_sec; err = bpf_map_alloc_id(map); if (err) goto free_map_sec; bpf_map_save_memcg(map); + bpf_token_put(token); err = bpf_map_new_fd(map, f_flags); if (err < 0) { @@ -1287,6 +1341,8 @@ static int map_create(union bpf_attr *attr) free_map: btf_put(map->btf); map->ops->map_free(map); +put_token: + bpf_token_put(token); return err; } @@ -1524,6 +1580,8 @@ static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr) } err = bpf_map_update_value(map, f.file, key, value, attr->flags); + if (!err) + maybe_wait_bpf_programs(map); kvfree(value); free_key: @@ -1579,7 +1637,8 @@ static int map_delete_elem(union bpf_attr *attr, bpfptr_t uattr) err = map->ops->map_delete_elem(map, key); rcu_read_unlock(); bpf_enable_instrumentation(); - maybe_wait_bpf_programs(map); + if (!err) + maybe_wait_bpf_programs(map); out: kvfree(key); err_put: @@ -1676,6 +1735,9 @@ int generic_map_delete_batch(struct bpf_map *map, if (!max_count) return 0; + if (put_user(0, &uattr->batch.count)) + return -EFAULT; + key = kvmalloc(map->key_size, GFP_USER | __GFP_NOWARN); if (!key) return -ENOMEM; @@ -1705,7 +1767,6 @@ int generic_map_delete_batch(struct bpf_map *map, kvfree(key); - maybe_wait_bpf_programs(map); return err; } @@ -1733,6 +1794,9 @@ int generic_map_update_batch(struct bpf_map *map, struct file *map_file, if (!max_count) return 0; + if (put_user(0, &uattr->batch.count)) + return -EFAULT; + key = kvmalloc(map->key_size, GFP_USER | __GFP_NOWARN); if (!key) return -ENOMEM; @@ -1763,6 +1827,7 @@ int generic_map_update_batch(struct bpf_map *map, struct file *map_file, kvfree(value); kvfree(key); + return err; } @@ -2108,7 +2173,7 @@ static void __bpf_prog_put_rcu(struct rcu_head *rcu) kvfree(aux->func_info); kfree(aux->func_info_aux); free_uid(aux->user); - security_bpf_prog_free(aux); + security_bpf_prog_free(aux->prog); bpf_prog_free(aux->prog); } @@ -2554,13 +2619,15 @@ static bool is_perfmon_prog_type(enum bpf_prog_type prog_type) } /* last field in 'union bpf_attr' used by this command */ -#define BPF_PROG_LOAD_LAST_FIELD log_true_size +#define BPF_PROG_LOAD_LAST_FIELD prog_token_fd static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) { enum bpf_prog_type type = attr->prog_type; struct bpf_prog *prog, *dst_prog = NULL; struct btf *attach_btf = NULL; + struct bpf_token *token = NULL; + bool bpf_cap; int err; char license[128]; @@ -2577,10 +2644,31 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) BPF_F_TEST_REG_INVARIANTS)) return -EINVAL; + bpf_prog_load_fixup_attach_type(attr); + + if (attr->prog_token_fd) { + token = bpf_token_get_from_fd(attr->prog_token_fd); + if (IS_ERR(token)) + return PTR_ERR(token); + /* if current token doesn't grant prog loading permissions, + * then we can't use this token, so ignore it and rely on + * system-wide capabilities checks + */ + if (!bpf_token_allow_cmd(token, BPF_PROG_LOAD) || + !bpf_token_allow_prog_type(token, attr->prog_type, + attr->expected_attach_type)) { + bpf_token_put(token); + token = NULL; + } + } + + bpf_cap = bpf_token_capable(token, CAP_BPF); + err = -EPERM; + if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && (attr->prog_flags & BPF_F_ANY_ALIGNMENT) && - !bpf_capable()) - return -EPERM; + !bpf_cap) + goto put_token; /* Intent here is for unprivileged_bpf_disabled to block BPF program * creation for unprivileged users; other actions depend @@ -2589,21 +2677,23 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) * capability checks are still carried out for these * and other operations. */ - if (sysctl_unprivileged_bpf_disabled && !bpf_capable()) - return -EPERM; + if (sysctl_unprivileged_bpf_disabled && !bpf_cap) + goto put_token; if (attr->insn_cnt == 0 || - attr->insn_cnt > (bpf_capable() ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS)) - return -E2BIG; + attr->insn_cnt > (bpf_cap ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS)) { + err = -E2BIG; + goto put_token; + } if (type != BPF_PROG_TYPE_SOCKET_FILTER && type != BPF_PROG_TYPE_CGROUP_SKB && - !bpf_capable()) - return -EPERM; + !bpf_cap) + goto put_token; - if (is_net_admin_prog_type(type) && !capable(CAP_NET_ADMIN) && !capable(CAP_SYS_ADMIN)) - return -EPERM; - if (is_perfmon_prog_type(type) && !perfmon_capable()) - return -EPERM; + if (is_net_admin_prog_type(type) && !bpf_token_capable(token, CAP_NET_ADMIN)) + goto put_token; + if (is_perfmon_prog_type(type) && !bpf_token_capable(token, CAP_PERFMON)) + goto put_token; /* attach_prog_fd/attach_btf_obj_fd can specify fd of either bpf_prog * or btf, we need to check which one it is @@ -2613,27 +2703,33 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) if (IS_ERR(dst_prog)) { dst_prog = NULL; attach_btf = btf_get_by_fd(attr->attach_btf_obj_fd); - if (IS_ERR(attach_btf)) - return -EINVAL; + if (IS_ERR(attach_btf)) { + err = -EINVAL; + goto put_token; + } if (!btf_is_kernel(attach_btf)) { /* attaching through specifying bpf_prog's BTF * objects directly might be supported eventually */ btf_put(attach_btf); - return -ENOTSUPP; + err = -ENOTSUPP; + goto put_token; } } } else if (attr->attach_btf_id) { /* fall back to vmlinux BTF, if BTF type ID is specified */ attach_btf = bpf_get_btf_vmlinux(); - if (IS_ERR(attach_btf)) - return PTR_ERR(attach_btf); - if (!attach_btf) - return -EINVAL; + if (IS_ERR(attach_btf)) { + err = PTR_ERR(attach_btf); + goto put_token; + } + if (!attach_btf) { + err = -EINVAL; + goto put_token; + } btf_get(attach_btf); } - bpf_prog_load_fixup_attach_type(attr); if (bpf_prog_load_check_attach(type, attr->expected_attach_type, attach_btf, attr->attach_btf_id, dst_prog)) { @@ -2641,7 +2737,8 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) bpf_prog_put(dst_prog); if (attach_btf) btf_put(attach_btf); - return -EINVAL; + err = -EINVAL; + goto put_token; } /* plain bpf_prog allocation */ @@ -2651,7 +2748,8 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) bpf_prog_put(dst_prog); if (attach_btf) btf_put(attach_btf); - return -ENOMEM; + err = -EINVAL; + goto put_token; } prog->expected_attach_type = attr->expected_attach_type; @@ -2662,9 +2760,9 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) prog->aux->sleepable = attr->prog_flags & BPF_F_SLEEPABLE; prog->aux->xdp_has_frags = attr->prog_flags & BPF_F_XDP_HAS_FRAGS; - err = security_bpf_prog_alloc(prog->aux); - if (err) - goto free_prog; + /* move token into prog->aux, reuse taken refcnt */ + prog->aux->token = token; + token = NULL; prog->aux->user = get_current_user(); prog->len = attr->insn_cnt; @@ -2673,12 +2771,12 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) if (copy_from_bpfptr(prog->insns, make_bpfptr(attr->insns, uattr.is_kernel), bpf_prog_insn_size(prog)) != 0) - goto free_prog_sec; + goto free_prog; /* copy eBPF program license from user space */ if (strncpy_from_bpfptr(license, make_bpfptr(attr->license, uattr.is_kernel), sizeof(license) - 1) < 0) - goto free_prog_sec; + goto free_prog; license[sizeof(license) - 1] = 0; /* eBPF programs must be GPL compatible to use GPL-ed functions */ @@ -2692,25 +2790,29 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) if (bpf_prog_is_dev_bound(prog->aux)) { err = bpf_prog_dev_bound_init(prog, attr); if (err) - goto free_prog_sec; + goto free_prog; } if (type == BPF_PROG_TYPE_EXT && dst_prog && bpf_prog_is_dev_bound(dst_prog->aux)) { err = bpf_prog_dev_bound_inherit(prog, dst_prog); if (err) - goto free_prog_sec; + goto free_prog; } /* find program type: socket_filter vs tracing_filter */ err = find_prog_type(type, prog); if (err < 0) - goto free_prog_sec; + goto free_prog; prog->aux->load_time = ktime_get_boottime_ns(); err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name, sizeof(attr->prog_name)); if (err < 0) + goto free_prog; + + err = security_bpf_prog_load(prog, attr, token); + if (err) goto free_prog_sec; /* run eBPF verifier */ @@ -2756,13 +2858,16 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) */ __bpf_prog_put_noref(prog, prog->aux->real_func_cnt); return err; + free_prog_sec: - free_uid(prog->aux->user); - security_bpf_prog_free(prog->aux); + security_bpf_prog_free(prog); free_prog: + free_uid(prog->aux->user); if (prog->aux->attach_btf) btf_put(prog->aux->attach_btf); bpf_prog_free(prog); +put_token: + bpf_token_put(token); return err; } @@ -3752,7 +3857,7 @@ static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog, case BPF_PROG_TYPE_SK_LOOKUP: return attach_type == prog->expected_attach_type ? 0 : -EINVAL; case BPF_PROG_TYPE_CGROUP_SKB: - if (!capable(CAP_NET_ADMIN)) + if (!bpf_token_capable(prog->aux->token, CAP_NET_ADMIN)) /* cg-skb progs can be loaded by unpriv user. * check permissions at attach time. */ @@ -3955,7 +4060,7 @@ static int bpf_prog_detach(const union bpf_attr *attr) static int bpf_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr) { - if (!capable(CAP_NET_ADMIN)) + if (!bpf_net_capable()) return -EPERM; if (CHECK_ATTR(BPF_PROG_QUERY)) return -EINVAL; @@ -4723,15 +4828,31 @@ static int bpf_obj_get_info_by_fd(const union bpf_attr *attr, return err; } -#define BPF_BTF_LOAD_LAST_FIELD btf_log_true_size +#define BPF_BTF_LOAD_LAST_FIELD btf_token_fd static int bpf_btf_load(const union bpf_attr *attr, bpfptr_t uattr, __u32 uattr_size) { + struct bpf_token *token = NULL; + if (CHECK_ATTR(BPF_BTF_LOAD)) return -EINVAL; - if (!bpf_capable()) + if (attr->btf_token_fd) { + token = bpf_token_get_from_fd(attr->btf_token_fd); + if (IS_ERR(token)) + return PTR_ERR(token); + if (!bpf_token_allow_cmd(token, BPF_BTF_LOAD)) { + bpf_token_put(token); + token = NULL; + } + } + + if (!bpf_token_capable(token, CAP_BPF)) { + bpf_token_put(token); return -EPERM; + } + + bpf_token_put(token); return btf_new_fd(attr, uattr, uattr_size); } @@ -4920,8 +5041,10 @@ static int bpf_map_do_batch(const union bpf_attr *attr, else BPF_DO_BATCH(map->ops->map_delete_batch, map, attr, uattr); err_put: - if (has_write) + if (has_write) { + maybe_wait_bpf_programs(map); bpf_map_write_active_dec(map); + } fdput(f); return err; } @@ -5323,6 +5446,11 @@ static int bpf_prog_bind_map(union bpf_attr *attr) goto out_unlock; } + /* The bpf program will not access the bpf map, but for the sake of + * simplicity, increase sleepable_refcnt for sleepable program as well. + */ + if (prog->aux->sleepable) + atomic64_inc(&map->sleepable_refcnt); memcpy(used_maps_new, used_maps_old, sizeof(used_maps_old[0]) * prog->aux->used_map_cnt); used_maps_new[prog->aux->used_map_cnt] = map; @@ -5342,6 +5470,20 @@ static int bpf_prog_bind_map(union bpf_attr *attr) return ret; } +#define BPF_TOKEN_CREATE_LAST_FIELD token_create.bpffs_fd + +static int token_create(union bpf_attr *attr) +{ + if (CHECK_ATTR(BPF_TOKEN_CREATE)) + return -EINVAL; + + /* no flags are supported yet */ + if (attr->token_create.flags) + return -EINVAL; + + return bpf_token_create(attr); +} + static int __sys_bpf(int cmd, bpfptr_t uattr, unsigned int size) { union bpf_attr attr; @@ -5475,6 +5617,9 @@ static int __sys_bpf(int cmd, bpfptr_t uattr, unsigned int size) case BPF_PROG_BIND_MAP: err = bpf_prog_bind_map(&attr); break; + case BPF_TOKEN_CREATE: + err = token_create(&attr); + break; default: err = -EINVAL; break; @@ -5581,7 +5726,7 @@ static const struct bpf_func_proto bpf_sys_bpf_proto = { const struct bpf_func_proto * __weak tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { - return bpf_base_func_proto(func_id); + return bpf_base_func_proto(func_id, prog); } BPF_CALL_1(bpf_sys_close, u32, fd) @@ -5631,7 +5776,8 @@ syscall_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { switch (func_id) { case BPF_FUNC_sys_bpf: - return !perfmon_capable() ? NULL : &bpf_sys_bpf_proto; + return !bpf_token_capable(prog->aux->token, CAP_PERFMON) + ? NULL : &bpf_sys_bpf_proto; case BPF_FUNC_btf_find_by_name_kind: return &bpf_btf_find_by_name_kind_proto; case BPF_FUNC_sys_close: diff --git a/kernel/bpf/tnum.c b/kernel/bpf/tnum.c index f4c91c9b27d7f1..9dbc31b25e3d08 100644 --- a/kernel/bpf/tnum.c +++ b/kernel/bpf/tnum.c @@ -172,12 +172,6 @@ bool tnum_in(struct tnum a, struct tnum b) return a.value == b.value; } -int tnum_strn(char *str, size_t size, struct tnum a) -{ - return snprintf(str, size, "(%#llx; %#llx)", a.value, a.mask); -} -EXPORT_SYMBOL_GPL(tnum_strn); - int tnum_sbin(char *str, size_t size, struct tnum a) { size_t n; diff --git a/kernel/bpf/token.c b/kernel/bpf/token.c new file mode 100644 index 00000000000000..a86fccd57e2d0c --- /dev/null +++ b/kernel/bpf/token.c @@ -0,0 +1,271 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +bool bpf_token_capable(const struct bpf_token *token, int cap) +{ + /* BPF token allows ns_capable() level of capabilities, but only if + * token's userns is *exactly* the same as current user's userns + */ + if (token && current_user_ns() == token->userns) { + if (ns_capable(token->userns, cap) || + (cap != CAP_SYS_ADMIN && ns_capable(token->userns, CAP_SYS_ADMIN))) + return security_bpf_token_capable(token, cap) == 0; + } + /* otherwise fallback to capable() checks */ + return capable(cap) || (cap != CAP_SYS_ADMIN && capable(CAP_SYS_ADMIN)); +} + +void bpf_token_inc(struct bpf_token *token) +{ + atomic64_inc(&token->refcnt); +} + +static void bpf_token_free(struct bpf_token *token) +{ + security_bpf_token_free(token); + put_user_ns(token->userns); + kvfree(token); +} + +static void bpf_token_put_deferred(struct work_struct *work) +{ + struct bpf_token *token = container_of(work, struct bpf_token, work); + + bpf_token_free(token); +} + +void bpf_token_put(struct bpf_token *token) +{ + if (!token) + return; + + if (!atomic64_dec_and_test(&token->refcnt)) + return; + + INIT_WORK(&token->work, bpf_token_put_deferred); + schedule_work(&token->work); +} + +static int bpf_token_release(struct inode *inode, struct file *filp) +{ + struct bpf_token *token = filp->private_data; + + bpf_token_put(token); + return 0; +} + +static void bpf_token_show_fdinfo(struct seq_file *m, struct file *filp) +{ + struct bpf_token *token = filp->private_data; + u64 mask; + + BUILD_BUG_ON(__MAX_BPF_CMD >= 64); + mask = (1ULL << __MAX_BPF_CMD) - 1; + if ((token->allowed_cmds & mask) == mask) + seq_printf(m, "allowed_cmds:\tany\n"); + else + seq_printf(m, "allowed_cmds:\t0x%llx\n", token->allowed_cmds); + + BUILD_BUG_ON(__MAX_BPF_MAP_TYPE >= 64); + mask = (1ULL << __MAX_BPF_MAP_TYPE) - 1; + if ((token->allowed_maps & mask) == mask) + seq_printf(m, "allowed_maps:\tany\n"); + else + seq_printf(m, "allowed_maps:\t0x%llx\n", token->allowed_maps); + + BUILD_BUG_ON(__MAX_BPF_PROG_TYPE >= 64); + mask = (1ULL << __MAX_BPF_PROG_TYPE) - 1; + if ((token->allowed_progs & mask) == mask) + seq_printf(m, "allowed_progs:\tany\n"); + else + seq_printf(m, "allowed_progs:\t0x%llx\n", token->allowed_progs); + + BUILD_BUG_ON(__MAX_BPF_ATTACH_TYPE >= 64); + mask = (1ULL << __MAX_BPF_ATTACH_TYPE) - 1; + if ((token->allowed_attachs & mask) == mask) + seq_printf(m, "allowed_attachs:\tany\n"); + else + seq_printf(m, "allowed_attachs:\t0x%llx\n", token->allowed_attachs); +} + +#define BPF_TOKEN_INODE_NAME "bpf-token" + +static const struct inode_operations bpf_token_iops = { }; + +static const struct file_operations bpf_token_fops = { + .release = bpf_token_release, + .show_fdinfo = bpf_token_show_fdinfo, +}; + +int bpf_token_create(union bpf_attr *attr) +{ + struct bpf_mount_opts *mnt_opts; + struct bpf_token *token = NULL; + struct user_namespace *userns; + struct inode *inode; + struct file *file; + struct path path; + struct fd f; + umode_t mode; + int err, fd; + + f = fdget(attr->token_create.bpffs_fd); + if (!f.file) + return -EBADF; + + path = f.file->f_path; + path_get(&path); + fdput(f); + + if (path.dentry != path.mnt->mnt_sb->s_root) { + err = -EINVAL; + goto out_path; + } + if (path.mnt->mnt_sb->s_op != &bpf_super_ops) { + err = -EINVAL; + goto out_path; + } + err = path_permission(&path, MAY_ACCESS); + if (err) + goto out_path; + + userns = path.dentry->d_sb->s_user_ns; + /* + * Enforce that creators of BPF tokens are in the same user + * namespace as the BPF FS instance. This makes reasoning about + * permissions a lot easier and we can always relax this later. + */ + if (current_user_ns() != userns) { + err = -EPERM; + goto out_path; + } + if (!ns_capable(userns, CAP_BPF)) { + err = -EPERM; + goto out_path; + } + + mnt_opts = path.dentry->d_sb->s_fs_info; + if (mnt_opts->delegate_cmds == 0 && + mnt_opts->delegate_maps == 0 && + mnt_opts->delegate_progs == 0 && + mnt_opts->delegate_attachs == 0) { + err = -ENOENT; /* no BPF token delegation is set up */ + goto out_path; + } + + mode = S_IFREG | ((S_IRUSR | S_IWUSR) & ~current_umask()); + inode = bpf_get_inode(path.mnt->mnt_sb, NULL, mode); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + goto out_path; + } + + inode->i_op = &bpf_token_iops; + inode->i_fop = &bpf_token_fops; + clear_nlink(inode); /* make sure it is unlinked */ + + file = alloc_file_pseudo(inode, path.mnt, BPF_TOKEN_INODE_NAME, O_RDWR, &bpf_token_fops); + if (IS_ERR(file)) { + iput(inode); + err = PTR_ERR(file); + goto out_path; + } + + token = kvzalloc(sizeof(*token), GFP_USER); + if (!token) { + err = -ENOMEM; + goto out_file; + } + + atomic64_set(&token->refcnt, 1); + + /* remember bpffs owning userns for future ns_capable() checks */ + token->userns = get_user_ns(userns); + + token->allowed_cmds = mnt_opts->delegate_cmds; + token->allowed_maps = mnt_opts->delegate_maps; + token->allowed_progs = mnt_opts->delegate_progs; + token->allowed_attachs = mnt_opts->delegate_attachs; + + err = security_bpf_token_create(token, attr, &path); + if (err) + goto out_token; + + fd = get_unused_fd_flags(O_CLOEXEC); + if (fd < 0) { + err = fd; + goto out_token; + } + + file->private_data = token; + fd_install(fd, file); + + path_put(&path); + return fd; + +out_token: + bpf_token_free(token); +out_file: + fput(file); +out_path: + path_put(&path); + return err; +} + +struct bpf_token *bpf_token_get_from_fd(u32 ufd) +{ + struct fd f = fdget(ufd); + struct bpf_token *token; + + if (!f.file) + return ERR_PTR(-EBADF); + if (f.file->f_op != &bpf_token_fops) { + fdput(f); + return ERR_PTR(-EINVAL); + } + + token = f.file->private_data; + bpf_token_inc(token); + fdput(f); + + return token; +} + +bool bpf_token_allow_cmd(const struct bpf_token *token, enum bpf_cmd cmd) +{ + /* BPF token can be used only within exactly the same userns in which + * it was created + */ + if (!token || current_user_ns() != token->userns) + return false; + if (!(token->allowed_cmds & (1ULL << cmd))) + return false; + return security_bpf_token_cmd(token, cmd) == 0; +} + +bool bpf_token_allow_map_type(const struct bpf_token *token, enum bpf_map_type type) +{ + if (!token || type >= __MAX_BPF_MAP_TYPE) + return false; + + return token->allowed_maps & (1ULL << type); +} + +bool bpf_token_allow_prog_type(const struct bpf_token *token, + enum bpf_prog_type prog_type, + enum bpf_attach_type attach_type) +{ + if (!token || prog_type >= __MAX_BPF_PROG_TYPE || attach_type >= __MAX_BPF_ATTACH_TYPE) + return false; + + return (token->allowed_progs & (1ULL << prog_type)) && + (token->allowed_attachs & (1ULL << attach_type)); +} diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index e97aeda3a86b55..d382f5ebe06c8f 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -115,10 +115,10 @@ bool bpf_prog_has_trampoline(const struct bpf_prog *prog) (ptype == BPF_PROG_TYPE_LSM && eatype == BPF_LSM_MAC); } -void bpf_image_ksym_add(void *data, struct bpf_ksym *ksym) +void bpf_image_ksym_add(void *data, unsigned int size, struct bpf_ksym *ksym) { ksym->start = (unsigned long) data; - ksym->end = ksym->start + PAGE_SIZE; + ksym->end = ksym->start + size; bpf_ksym_add(ksym); perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_BPF, ksym->start, PAGE_SIZE, false, ksym->name); @@ -254,8 +254,8 @@ bpf_trampoline_get_progs(const struct bpf_trampoline *tr, int *total, bool *ip_a static void bpf_tramp_image_free(struct bpf_tramp_image *im) { bpf_image_ksym_del(&im->ksym); - bpf_jit_free_exec(im->image); - bpf_jit_uncharge_modmem(PAGE_SIZE); + arch_free_bpf_trampoline(im->image, im->size); + bpf_jit_uncharge_modmem(im->size); percpu_ref_exit(&im->pcref); kfree_rcu(im, rcu); } @@ -349,7 +349,7 @@ static void bpf_tramp_image_put(struct bpf_tramp_image *im) call_rcu_tasks_trace(&im->rcu, __bpf_tramp_image_put_rcu_tasks); } -static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key) +static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, int size) { struct bpf_tramp_image *im; struct bpf_ksym *ksym; @@ -360,15 +360,15 @@ static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key) if (!im) goto out; - err = bpf_jit_charge_modmem(PAGE_SIZE); + err = bpf_jit_charge_modmem(size); if (err) goto out_free_im; + im->size = size; err = -ENOMEM; - im->image = image = bpf_jit_alloc_exec(PAGE_SIZE); + im->image = image = arch_alloc_bpf_trampoline(size); if (!image) goto out_uncharge; - set_vm_flush_reset_perms(image); err = percpu_ref_init(&im->pcref, __bpf_tramp_image_release, 0, GFP_KERNEL); if (err) @@ -377,13 +377,13 @@ static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key) ksym = &im->ksym; INIT_LIST_HEAD_RCU(&ksym->lnode); snprintf(ksym->name, KSYM_NAME_LEN, "bpf_trampoline_%llu", key); - bpf_image_ksym_add(image, ksym); + bpf_image_ksym_add(image, size, ksym); return im; out_free_image: - bpf_jit_free_exec(im->image); + arch_free_bpf_trampoline(im->image, im->size); out_uncharge: - bpf_jit_uncharge_modmem(PAGE_SIZE); + bpf_jit_uncharge_modmem(size); out_free_im: kfree(im); out: @@ -396,7 +396,7 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mut struct bpf_tramp_links *tlinks; u32 orig_flags = tr->flags; bool ip_arg = false; - int err, total; + int err, total, size; tlinks = bpf_trampoline_get_progs(tr, &total, &ip_arg); if (IS_ERR(tlinks)) @@ -409,12 +409,6 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mut goto out; } - im = bpf_tramp_image_alloc(tr->key); - if (IS_ERR(im)) { - err = PTR_ERR(im); - goto out; - } - /* clear all bits except SHARE_IPMODIFY and TAIL_CALL_CTX */ tr->flags &= (BPF_TRAMP_F_SHARE_IPMODIFY | BPF_TRAMP_F_TAIL_CALL_CTX); @@ -438,13 +432,31 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mut tr->flags |= BPF_TRAMP_F_ORIG_STACK; #endif - err = arch_prepare_bpf_trampoline(im, im->image, im->image + PAGE_SIZE, + size = arch_bpf_trampoline_size(&tr->func.model, tr->flags, + tlinks, tr->func.addr); + if (size < 0) { + err = size; + goto out; + } + + if (size > PAGE_SIZE) { + err = -E2BIG; + goto out; + } + + im = bpf_tramp_image_alloc(tr->key, size); + if (IS_ERR(im)) { + err = PTR_ERR(im); + goto out; + } + + err = arch_prepare_bpf_trampoline(im, im->image, im->image + size, &tr->func.model, tr->flags, tlinks, tr->func.addr); if (err < 0) goto out_free; - set_memory_rox((long)im->image, 1); + arch_protect_bpf_trampoline(im->image, im->size); WARN_ON(tr->cur_image && total == 0); if (tr->cur_image) @@ -464,9 +476,8 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mut tr->fops->func = NULL; tr->fops->trampoline = 0; - /* reset im->image memory attr for arch_prepare_bpf_trampoline */ - set_memory_nx((long)im->image, 1); - set_memory_rw((long)im->image, 1); + /* free im memory and reallocate later */ + bpf_tramp_image_free(im); goto again; } #endif @@ -1032,10 +1043,50 @@ bpf_trampoline_exit_t bpf_trampoline_exit(const struct bpf_prog *prog) } int __weak -arch_prepare_bpf_trampoline(struct bpf_tramp_image *tr, void *image, void *image_end, +arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, const struct btf_func_model *m, u32 flags, struct bpf_tramp_links *tlinks, - void *orig_call) + void *func_addr) +{ + return -ENOTSUPP; +} + +void * __weak arch_alloc_bpf_trampoline(unsigned int size) +{ + void *image; + + if (WARN_ON_ONCE(size > PAGE_SIZE)) + return NULL; + image = bpf_jit_alloc_exec(PAGE_SIZE); + if (image) + set_vm_flush_reset_perms(image); + return image; +} + +void __weak arch_free_bpf_trampoline(void *image, unsigned int size) +{ + WARN_ON_ONCE(size > PAGE_SIZE); + /* bpf_jit_free_exec doesn't need "size", but + * bpf_prog_pack_free() needs it. + */ + bpf_jit_free_exec(image); +} + +void __weak arch_protect_bpf_trampoline(void *image, unsigned int size) +{ + WARN_ON_ONCE(size > PAGE_SIZE); + set_memory_rox((long)image, 1); +} + +void __weak arch_unprotect_bpf_trampoline(void *image, unsigned int size) +{ + WARN_ON_ONCE(size > PAGE_SIZE); + set_memory_nx((long)image, 1); + set_memory_rw((long)image, 1); +} + +int __weak arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, + struct bpf_tramp_links *tlinks, void *func_addr) { return -ENOTSUPP; } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 8e7b6072e3f4db..9456ee0ad129af 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -362,20 +362,23 @@ __printf(2, 3) static void verbose(void *private_data, const char *fmt, ...) static void verbose_invalid_scalar(struct bpf_verifier_env *env, struct bpf_reg_state *reg, - struct tnum *range, const char *ctx, + struct bpf_retval_range range, const char *ctx, const char *reg_name) { - char tn_buf[48]; + bool unknown = true; - verbose(env, "At %s the register %s ", ctx, reg_name); - if (!tnum_is_unknown(reg->var_off)) { - tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); - verbose(env, "has value %s", tn_buf); - } else { - verbose(env, "has unknown scalar value"); + verbose(env, "%s the register %s has", ctx, reg_name); + if (reg->smin_value > S64_MIN) { + verbose(env, " smin=%lld", reg->smin_value); + unknown = false; + } + if (reg->smax_value < S64_MAX) { + verbose(env, " smax=%lld", reg->smax_value); + unknown = false; } - tnum_strn(tn_buf, sizeof(tn_buf), *range); - verbose(env, " should have been in %s\n", tn_buf); + if (unknown) + verbose(env, " unknown scalar value"); + verbose(env, " should have been in [%d, %d]\n", range.minval, range.maxval); } static bool type_may_be_null(u32 type) @@ -439,6 +442,25 @@ static struct bpf_func_info_aux *subprog_aux(const struct bpf_verifier_env *env, return &env->prog->aux->func_info_aux[subprog]; } +static struct bpf_subprog_info *subprog_info(struct bpf_verifier_env *env, int subprog) +{ + return &env->subprog_info[subprog]; +} + +static void mark_subprog_exc_cb(struct bpf_verifier_env *env, int subprog) +{ + struct bpf_subprog_info *info = subprog_info(env, subprog); + + info->is_cb = true; + info->is_async_cb = true; + info->is_exception_cb = true; +} + +static bool subprog_is_exc_cb(struct bpf_verifier_env *env, int subprog) +{ + return subprog_info(env, subprog)->is_exception_cb; +} + static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg) { return btf_record_has_field(reg_btf_record(reg), BPF_SPIN_LOCK); @@ -1141,6 +1163,21 @@ static bool is_spilled_scalar_reg(const struct bpf_stack_state *stack) stack->spilled_ptr.type == SCALAR_VALUE; } +/* Mark stack slot as STACK_MISC, unless it is already STACK_INVALID, in which + * case they are equivalent, or it's STACK_ZERO, in which case we preserve + * more precise STACK_ZERO. + * Note, in uprivileged mode leaving STACK_INVALID is wrong, so we take + * env->allow_ptr_leaks into account and force STACK_MISC, if necessary. + */ +static void mark_stack_slot_misc(struct bpf_verifier_env *env, u8 *stype) +{ + if (*stype == STACK_ZERO) + return; + if (env->allow_ptr_leaks && *stype == STACK_INVALID) + return; + *stype = STACK_MISC; +} + static void scrub_spilled_slot(u8 *stype) { if (*stype != STACK_INVALID) @@ -1241,9 +1278,16 @@ static int resize_reference_state(struct bpf_func_state *state, size_t n) return 0; } -static int grow_stack_state(struct bpf_func_state *state, int size) +/* Possibly update state->allocated_stack to be at least size bytes. Also + * possibly update the function's high-water mark in its bpf_subprog_info. + */ +static int grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int size) { - size_t old_n = state->allocated_stack / BPF_REG_SIZE, n = size / BPF_REG_SIZE; + size_t old_n = state->allocated_stack / BPF_REG_SIZE, n; + + /* The stack size is always a multiple of BPF_REG_SIZE. */ + size = round_up(size, BPF_REG_SIZE); + n = size / BPF_REG_SIZE; if (old_n >= n) return 0; @@ -1253,6 +1297,11 @@ static int grow_stack_state(struct bpf_func_state *state, int size) return -ENOMEM; state->allocated_stack = size; + + /* update known max for given subprogram */ + if (env->subprog_info[state->subprogno].stack_depth < size) + env->subprog_info[state->subprogno].stack_depth = size; + return 0; } @@ -1352,8 +1401,8 @@ static int copy_verifier_state(struct bpf_verifier_state *dst_state, int i, err; dst_state->jmp_history = copy_array(dst_state->jmp_history, src->jmp_history, - src->jmp_history_cnt, sizeof(struct bpf_idx_pair), - GFP_USER); + src->jmp_history_cnt, sizeof(*dst_state->jmp_history), + GFP_USER); if (!dst_state->jmp_history) return -ENOMEM; dst_state->jmp_history_cnt = src->jmp_history_cnt; @@ -1728,10 +1777,14 @@ static void __mark_reg_known_zero(struct bpf_reg_state *reg) __mark_reg_known(reg, 0); } -static void __mark_reg_const_zero(struct bpf_reg_state *reg) +static void __mark_reg_const_zero(const struct bpf_verifier_env *env, struct bpf_reg_state *reg) { __mark_reg_known(reg, 0); reg->type = SCALAR_VALUE; + /* all scalars are assumed imprecise initially (unless unprivileged, + * in which case everything is forced to be precise) + */ + reg->precise = !env->bpf_capable; } static void mark_reg_known_zero(struct bpf_verifier_env *env, @@ -2305,6 +2358,11 @@ static void init_reg_state(struct bpf_verifier_env *env, regs[BPF_REG_FP].frameno = state->frameno; } +static struct bpf_retval_range retval_range(s32 minval, s32 maxval) +{ + return (struct bpf_retval_range){ minval, maxval }; +} + #define BPF_MAIN_FUNC (-1) static void init_func_state(struct bpf_verifier_env *env, struct bpf_func_state *state, @@ -2313,7 +2371,7 @@ static void init_func_state(struct bpf_verifier_env *env, state->callsite = callsite; state->frameno = frameno; state->subprogno = subprogno; - state->callback_ret_range = tnum_range(0, 0); + state->callback_ret_range = retval_range(0, 0); init_reg_state(env, state); mark_verifier_state_scratched(env); } @@ -2857,6 +2915,7 @@ static int add_subprog_and_kfunc(struct bpf_verifier_env *env) if (env->subprog_info[i].start != ex_cb_insn) continue; env->exception_callback_subprog = i; + mark_subprog_exc_cb(env, i); break; } } @@ -3213,6 +3272,21 @@ static int check_reg_arg(struct bpf_verifier_env *env, u32 regno, return __check_reg_arg(env, state->regs, regno, t); } +static int insn_stack_access_flags(int frameno, int spi) +{ + return INSN_F_STACK_ACCESS | (spi << INSN_F_SPI_SHIFT) | frameno; +} + +static int insn_stack_access_spi(int insn_flags) +{ + return (insn_flags >> INSN_F_SPI_SHIFT) & INSN_F_SPI_MASK; +} + +static int insn_stack_access_frameno(int insn_flags) +{ + return insn_flags & INSN_F_FRAMENO_MASK; +} + static void mark_jmp_point(struct bpf_verifier_env *env, int idx) { env->insn_aux_data[idx].jmp_point = true; @@ -3224,28 +3298,51 @@ static bool is_jmp_point(struct bpf_verifier_env *env, int insn_idx) } /* for any branch, call, exit record the history of jmps in the given state */ -static int push_jmp_history(struct bpf_verifier_env *env, - struct bpf_verifier_state *cur) +static int push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur, + int insn_flags) { u32 cnt = cur->jmp_history_cnt; - struct bpf_idx_pair *p; + struct bpf_jmp_history_entry *p; size_t alloc_size; - if (!is_jmp_point(env, env->insn_idx)) + /* combine instruction flags if we already recorded this instruction */ + if (env->cur_hist_ent) { + /* atomic instructions push insn_flags twice, for READ and + * WRITE sides, but they should agree on stack slot + */ + WARN_ONCE((env->cur_hist_ent->flags & insn_flags) && + (env->cur_hist_ent->flags & insn_flags) != insn_flags, + "verifier insn history bug: insn_idx %d cur flags %x new flags %x\n", + env->insn_idx, env->cur_hist_ent->flags, insn_flags); + env->cur_hist_ent->flags |= insn_flags; return 0; + } cnt++; alloc_size = kmalloc_size_roundup(size_mul(cnt, sizeof(*p))); p = krealloc(cur->jmp_history, alloc_size, GFP_USER); if (!p) return -ENOMEM; - p[cnt - 1].idx = env->insn_idx; - p[cnt - 1].prev_idx = env->prev_insn_idx; cur->jmp_history = p; + + p = &cur->jmp_history[cnt - 1]; + p->idx = env->insn_idx; + p->prev_idx = env->prev_insn_idx; + p->flags = insn_flags; cur->jmp_history_cnt = cnt; + env->cur_hist_ent = p; + return 0; } +static struct bpf_jmp_history_entry *get_jmp_hist_entry(struct bpf_verifier_state *st, + u32 hist_end, int insn_idx) +{ + if (hist_end > 0 && st->jmp_history[hist_end - 1].idx == insn_idx) + return &st->jmp_history[hist_end - 1]; + return NULL; +} + /* Backtrack one insn at a time. If idx is not at the top of recorded * history then previous instruction came from straight line execution. * Return -ENOENT if we exhausted all instructions within given state. @@ -3372,16 +3469,6 @@ static inline void bt_clear_frame_slot(struct backtrack_state *bt, u32 frame, u3 bt->stack_masks[frame] &= ~(1ull << slot); } -static inline void bt_set_slot(struct backtrack_state *bt, u32 slot) -{ - bt_set_frame_slot(bt, bt->frame, slot); -} - -static inline void bt_clear_slot(struct backtrack_state *bt, u32 slot) -{ - bt_clear_frame_slot(bt, bt->frame, slot); -} - static inline u32 bt_frame_reg_mask(struct backtrack_state *bt, u32 frame) { return bt->reg_masks[frame]; @@ -3407,9 +3494,9 @@ static inline bool bt_is_reg_set(struct backtrack_state *bt, u32 reg) return bt->reg_masks[bt->frame] & (1 << reg); } -static inline bool bt_is_slot_set(struct backtrack_state *bt, u32 slot) +static inline bool bt_is_frame_slot_set(struct backtrack_state *bt, u32 frame, u32 slot) { - return bt->stack_masks[bt->frame] & (1ull << slot); + return bt->stack_masks[frame] & (1ull << slot); } /* format registers bitmask, e.g., "r0,r2,r4" for 0x15 mask */ @@ -3463,7 +3550,7 @@ static bool calls_callback(struct bpf_verifier_env *env, int insn_idx); * - *was* processed previously during backtracking. */ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, - struct backtrack_state *bt) + struct bpf_jmp_history_entry *hist, struct backtrack_state *bt) { const struct bpf_insn_cbs cbs = { .cb_call = disasm_kfunc_name, @@ -3476,7 +3563,7 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, u8 mode = BPF_MODE(insn->code); u32 dreg = insn->dst_reg; u32 sreg = insn->src_reg; - u32 spi, i; + u32 spi, i, fr; if (insn->code == 0) return 0; @@ -3537,20 +3624,15 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, * by 'precise' mark in corresponding register of this state. * No further tracking necessary. */ - if (insn->src_reg != BPF_REG_FP) + if (!hist || !(hist->flags & INSN_F_STACK_ACCESS)) return 0; - /* dreg = *(u64 *)[fp - off] was a fill from the stack. * that [fp - off] slot contains scalar that needs to be * tracked with precision */ - spi = (-insn->off - 1) / BPF_REG_SIZE; - if (spi >= 64) { - verbose(env, "BUG spi %d\n", spi); - WARN_ONCE(1, "verifier backtracking bug"); - return -EFAULT; - } - bt_set_slot(bt, spi); + spi = insn_stack_access_spi(hist->flags); + fr = insn_stack_access_frameno(hist->flags); + bt_set_frame_slot(bt, fr, spi); } else if (class == BPF_STX || class == BPF_ST) { if (bt_is_reg_set(bt, dreg)) /* stx & st shouldn't be using _scalar_ dst_reg @@ -3559,17 +3641,13 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, */ return -ENOTSUPP; /* scalars can only be spilled into stack */ - if (insn->dst_reg != BPF_REG_FP) + if (!hist || !(hist->flags & INSN_F_STACK_ACCESS)) return 0; - spi = (-insn->off - 1) / BPF_REG_SIZE; - if (spi >= 64) { - verbose(env, "BUG spi %d\n", spi); - WARN_ONCE(1, "verifier backtracking bug"); - return -EFAULT; - } - if (!bt_is_slot_set(bt, spi)) + spi = insn_stack_access_spi(hist->flags); + fr = insn_stack_access_frameno(hist->flags); + if (!bt_is_frame_slot_set(bt, fr, spi)) return 0; - bt_clear_slot(bt, spi); + bt_clear_frame_slot(bt, fr, spi); if (class == BPF_STX) bt_set_reg(bt, sreg); } else if (class == BPF_JMP || class == BPF_JMP32) { @@ -3613,10 +3691,14 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, WARN_ONCE(1, "verifier backtracking bug"); return -EFAULT; } - /* we don't track register spills perfectly, - * so fallback to force-precise instead of failing */ - if (bt_stack_mask(bt) != 0) - return -ENOTSUPP; + /* we are now tracking register spills correctly, + * so any instance of leftover slots is a bug + */ + if (bt_stack_mask(bt) != 0) { + verbose(env, "BUG stack slots %llx\n", bt_stack_mask(bt)); + WARN_ONCE(1, "verifier backtracking bug (subprog leftover stack slots)"); + return -EFAULT; + } /* propagate r1-r5 to the caller */ for (i = BPF_REG_1; i <= BPF_REG_5; i++) { if (bt_is_reg_set(bt, i)) { @@ -3641,8 +3723,11 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, WARN_ONCE(1, "verifier backtracking bug"); return -EFAULT; } - if (bt_stack_mask(bt) != 0) - return -ENOTSUPP; + if (bt_stack_mask(bt) != 0) { + verbose(env, "BUG stack slots %llx\n", bt_stack_mask(bt)); + WARN_ONCE(1, "verifier backtracking bug (callback leftover stack slots)"); + return -EFAULT; + } /* clear r1-r5 in callback subprog's mask */ for (i = BPF_REG_1; i <= BPF_REG_5; i++) bt_clear_reg(bt, i); @@ -4079,6 +4164,7 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int regno) for (;;) { DECLARE_BITMAP(mask, 64); u32 history = st->jmp_history_cnt; + struct bpf_jmp_history_entry *hist; if (env->log.level & BPF_LOG_LEVEL2) { verbose(env, "mark_precise: frame%d: last_idx %d first_idx %d subseq_idx %d \n", @@ -4142,7 +4228,8 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int regno) err = 0; skip_first = false; } else { - err = backtrack_insn(env, i, subseq_idx, bt); + hist = get_jmp_hist_entry(st, history, i); + err = backtrack_insn(env, i, subseq_idx, hist, bt); } if (err == -ENOTSUPP) { mark_all_scalars_precise(env, env->cur_state); @@ -4195,22 +4282,10 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int regno) bitmap_from_u64(mask, bt_frame_stack_mask(bt, fr)); for_each_set_bit(i, mask, 64) { if (i >= func->allocated_stack / BPF_REG_SIZE) { - /* the sequence of instructions: - * 2: (bf) r3 = r10 - * 3: (7b) *(u64 *)(r3 -8) = r0 - * 4: (79) r4 = *(u64 *)(r10 -8) - * doesn't contain jmps. It's backtracked - * as a single block. - * During backtracking insn 3 is not recognized as - * stack access, so at the end of backtracking - * stack slot fp-8 is still marked in stack_mask. - * However the parent state may not have accessed - * fp-8 and it's "unallocated" stack space. - * In such case fallback to conservative. - */ - mark_all_scalars_precise(env, env->cur_state); - bt_reset(bt); - return 0; + verbose(env, "BUG backtracking (stack slot %d, total slots %d)\n", + i, func->allocated_stack / BPF_REG_SIZE); + WARN_ONCE(1, "verifier backtracking bug (stack slot out of bounds)"); + return -EFAULT; } if (!is_spilled_scalar_reg(&func->stack[i])) { @@ -4347,7 +4422,8 @@ static void copy_register_state(struct bpf_reg_state *dst, const struct bpf_reg_ dst->live = live; } -static void save_register_state(struct bpf_func_state *state, +static void save_register_state(struct bpf_verifier_env *env, + struct bpf_func_state *state, int spi, struct bpf_reg_state *reg, int size) { @@ -4362,7 +4438,7 @@ static void save_register_state(struct bpf_func_state *state, /* size < 8 bytes spill */ for (; i; i--) - scrub_spilled_slot(&state->stack[spi].slot_type[i - 1]); + mark_stack_slot_misc(env, &state->stack[spi].slot_type[i - 1]); } static bool is_bpf_st_mem(struct bpf_insn *insn) @@ -4383,16 +4459,13 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err; struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; struct bpf_reg_state *reg = NULL; - u32 dst_reg = insn->dst_reg; + int insn_flags = insn_stack_access_flags(state->frameno, spi); - err = grow_stack_state(state, round_up(slot + 1, BPF_REG_SIZE)); - if (err) - return err; /* caller checked that off % size == 0 and -MAX_BPF_STACK <= off < 0, * so it's aligned access and [off, off + size) are within stack limits */ if (!env->allow_ptr_leaks && - state->stack[spi].slot_type[0] == STACK_SPILL && + is_spilled_reg(&state->stack[spi]) && size != BPF_REG_SIZE) { verbose(env, "attempt to corrupt spilled pointer on stack\n"); return -EACCES; @@ -4422,20 +4495,8 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, return err; mark_stack_slot_scratched(env, spi); - if (reg && !(off % BPF_REG_SIZE) && register_is_bounded(reg) && - !register_is_null(reg) && env->bpf_capable) { - if (dst_reg != BPF_REG_FP) { - /* The backtracking logic can only recognize explicit - * stack slot address like [fp - 8]. Other spill of - * scalar via different register has to be conservative. - * Backtrack from here and mark all registers as precise - * that contributed into 'reg' being a constant. - */ - err = mark_chain_precision(env, value_regno); - if (err) - return err; - } - save_register_state(state, spi, reg, size); + if (reg && !(off % BPF_REG_SIZE) && register_is_bounded(reg) && env->bpf_capable) { + save_register_state(env, state, spi, reg, size); /* Break the relation on a narrowing spill. */ if (fls64(reg->umax_value) > BITS_PER_BYTE * size) state->stack[spi].spilled_ptr.id = 0; @@ -4445,7 +4506,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, __mark_reg_known(&fake_reg, insn->imm); fake_reg.type = SCALAR_VALUE; - save_register_state(state, spi, &fake_reg, size); + save_register_state(env, state, spi, &fake_reg, size); } else if (reg && is_spillable_regtype(reg->type)) { /* register containing pointer is being spilled into stack */ if (size != BPF_REG_SIZE) { @@ -4457,7 +4518,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, verbose(env, "cannot spill pointers to stack into stack frame of the caller\n"); return -EINVAL; } - save_register_state(state, spi, reg, size); + save_register_state(env, state, spi, reg, size); } else { u8 type = STACK_MISC; @@ -4482,7 +4543,12 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, /* when we zero initialize stack slots mark them as such */ if ((reg && register_is_null(reg)) || (!reg && is_bpf_st_mem(insn) && insn->imm == 0)) { - /* backtracking doesn't work for STACK_ZERO yet. */ + /* STACK_ZERO case happened because register spill + * wasn't properly aligned at the stack slot boundary, + * so it's not a register spill anymore; force + * originating register to be precise to make + * STACK_ZERO correct for subsequent states + */ err = mark_chain_precision(env, value_regno); if (err) return err; @@ -4491,9 +4557,12 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, /* Mark slots affected by this stack write. */ for (i = 0; i < size; i++) - state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] = - type; + state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] = type; + insn_flags = 0; /* not a register spill */ } + + if (insn_flags) + return push_jmp_history(env, env->cur_state, insn_flags); return 0; } @@ -4543,10 +4612,6 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env, (!value_reg && is_bpf_st_mem(insn) && insn->imm == 0)) writing_zero = true; - err = grow_stack_state(state, round_up(-min_off, BPF_REG_SIZE)); - if (err) - return err; - for (i = min_off; i < max_off; i++) { int spi; @@ -4645,21 +4710,10 @@ static void mark_reg_stack_read(struct bpf_verifier_env *env, zeros++; } if (zeros == max_off - min_off) { - /* any access_size read into register is zero extended, - * so the whole register == const_zero + /* Any access_size read into register is zero extended, + * so the whole register == const_zero. */ - __mark_reg_const_zero(&state->regs[dst_regno]); - /* backtracking doesn't support STACK_ZERO yet, - * so mark it precise here, so that later - * backtracking can stop here. - * Backtracking may not need this if this register - * doesn't participate in pointer adjustment. - * Forward propagation of precise flag is not - * necessary either. This mark is only to stop - * backtracking. Any register that contributed - * to const 0 was marked precise before spill. - */ - state->regs[dst_regno].precise = true; + __mark_reg_const_zero(env, &state->regs[dst_regno]); } else { /* have read misc data from the stack */ mark_reg_unknown(env, state->regs, dst_regno); @@ -4686,6 +4740,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, int i, slot = -off - 1, spi = slot / BPF_REG_SIZE; struct bpf_reg_state *reg; u8 *stype, type; + int insn_flags = insn_stack_access_flags(reg_state->frameno, spi); stype = reg_state->stack[spi].slot_type; reg = ®_state->stack[spi].spilled_ptr; @@ -4718,25 +4773,42 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, copy_register_state(&state->regs[dst_regno], reg); state->regs[dst_regno].subreg_def = subreg_def; } else { + int spill_cnt = 0, zero_cnt = 0; + for (i = 0; i < size; i++) { type = stype[(slot - i) % BPF_REG_SIZE]; - if (type == STACK_SPILL) + if (type == STACK_SPILL) { + spill_cnt++; continue; + } if (type == STACK_MISC) continue; + if (type == STACK_ZERO) { + zero_cnt++; + continue; + } if (type == STACK_INVALID && env->allow_uninit_stack) continue; verbose(env, "invalid read from stack off %d+%d size %d\n", off, i, size); return -EACCES; } - mark_reg_unknown(env, state->regs, dst_regno); + + if (spill_cnt == size && + tnum_is_const(reg->var_off) && reg->var_off.value == 0) { + __mark_reg_const_zero(env, &state->regs[dst_regno]); + /* this IS register fill, so keep insn_flags */ + } else if (zero_cnt == size) { + /* similarly to mark_reg_stack_read(), preserve zeroes */ + __mark_reg_const_zero(env, &state->regs[dst_regno]); + insn_flags = 0; /* not restoring original register state */ + } else { + mark_reg_unknown(env, state->regs, dst_regno); + insn_flags = 0; /* not restoring original register state */ + } } state->regs[dst_regno].live |= REG_LIVE_WRITTEN; - return 0; - } - - if (dst_regno >= 0) { + } else if (dst_regno >= 0) { /* restore register state from stack */ copy_register_state(&state->regs[dst_regno], reg); /* mark reg as written since spilled pointer state likely @@ -4772,7 +4844,10 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); if (dst_regno >= 0) mark_reg_stack_read(env, reg_state, off, off + size, dst_regno); + insn_flags = 0; /* we are not restoring spilled register */ } + if (insn_flags) + return push_jmp_history(env, env->cur_state, insn_flags); return 0; } @@ -5701,20 +5776,6 @@ static int check_ptr_alignment(struct bpf_verifier_env *env, strict); } -static int update_stack_depth(struct bpf_verifier_env *env, - const struct bpf_func_state *func, - int off) -{ - u16 stack = env->subprog_info[func->subprogno].stack_depth; - - if (stack >= -off) - return 0; - - /* update known max for given subprogram */ - env->subprog_info[func->subprogno].stack_depth = -off; - return 0; -} - /* starting from main bpf function walk all instructions of the function * and recursively walk all callees that given function can call. * Ignore jump and exit insns. @@ -6504,13 +6565,14 @@ static int check_ptr_to_map_access(struct bpf_verifier_env *env, * The minimum valid offset is -MAX_BPF_STACK for writes, and * -state->allocated_stack for reads. */ -static int check_stack_slot_within_bounds(int off, - struct bpf_func_state *state, - enum bpf_access_type t) +static int check_stack_slot_within_bounds(struct bpf_verifier_env *env, + s64 off, + struct bpf_func_state *state, + enum bpf_access_type t) { int min_valid_off; - if (t == BPF_WRITE) + if (t == BPF_WRITE || env->allow_uninit_stack) min_valid_off = -MAX_BPF_STACK; else min_valid_off = -state->allocated_stack; @@ -6533,7 +6595,7 @@ static int check_stack_access_within_bounds( struct bpf_reg_state *regs = cur_regs(env); struct bpf_reg_state *reg = regs + regno; struct bpf_func_state *state = func(env, reg); - int min_off, max_off; + s64 min_off, max_off; int err; char *err_extra; @@ -6546,11 +6608,8 @@ static int check_stack_access_within_bounds( err_extra = " write to"; if (tnum_is_const(reg->var_off)) { - min_off = reg->var_off.value + off; - if (access_size > 0) - max_off = min_off + access_size - 1; - else - max_off = min_off; + min_off = (s64)reg->var_off.value + off; + max_off = min_off + access_size; } else { if (reg->smax_value >= BPF_MAX_VAR_OFF || reg->smin_value <= -BPF_MAX_VAR_OFF) { @@ -6559,15 +6618,12 @@ static int check_stack_access_within_bounds( return -EACCES; } min_off = reg->smin_value + off; - if (access_size > 0) - max_off = reg->smax_value + off + access_size - 1; - else - max_off = min_off; + max_off = reg->smax_value + off + access_size; } - err = check_stack_slot_within_bounds(min_off, state, type); - if (!err) - err = check_stack_slot_within_bounds(max_off, state, type); + err = check_stack_slot_within_bounds(env, min_off, state, type); + if (!err && max_off > 0) + err = -EINVAL; /* out of stack access into non-negative offsets */ if (err) { if (tnum_is_const(reg->var_off)) { @@ -6577,11 +6633,16 @@ static int check_stack_access_within_bounds( char tn_buf[48]; tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); - verbose(env, "invalid variable-offset%s stack R%d var_off=%s size=%d\n", - err_extra, regno, tn_buf, access_size); + verbose(env, "invalid variable-offset%s stack R%d var_off=%s off=%d size=%d\n", + err_extra, regno, tn_buf, off, access_size); } + return err; } - return err; + + /* Note that there is no stack access with offset zero, so the needed stack + * size is -min_off, not -min_off+1. + */ + return grow_stack_state(env, state, -min_off /* size */); } /* check whether memory at (regno + off) is accessible for t = (read | write) @@ -6596,7 +6657,6 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn { struct bpf_reg_state *regs = cur_regs(env); struct bpf_reg_state *reg = regs + regno; - struct bpf_func_state *state; int size, err = 0; size = bpf_size_to_bytes(bpf_size); @@ -6739,11 +6799,6 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn if (err) return err; - state = func(env, reg); - err = update_stack_depth(env, state, off); - if (err) - return err; - if (t == BPF_READ) err = check_stack_read(env, regno, off, size, value_regno); @@ -6932,13 +6987,13 @@ static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_i BPF_SIZE(insn->code), BPF_WRITE, -1, true, false); if (err) return err; - return 0; } /* When register 'regno' is used to read the stack (either directly or through * a helper function) make sure that it's within stack boundary and, depending - * on the access type, that all elements of the stack are initialized. + * on the access type and privileges, that all elements of the stack are + * initialized. * * 'off' includes 'regno->off', but not its dynamic part (if any). * @@ -7046,8 +7101,11 @@ static int check_stack_range_initialized( slot = -i - 1; spi = slot / BPF_REG_SIZE; - if (state->allocated_stack <= slot) - goto err; + if (state->allocated_stack <= slot) { + verbose(env, "verifier bug: allocated_stack too small"); + return -EFAULT; + } + stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; if (*stype == STACK_MISC) goto mark; @@ -7071,7 +7129,6 @@ static int check_stack_range_initialized( goto mark; } -err: if (tnum_is_const(reg->var_off)) { verbose(env, "invalid%s read from stack R%d off %d+%d size %d\n", err_extra, regno, min_off, i - min_off, access_size); @@ -7096,7 +7153,7 @@ static int check_stack_range_initialized( * helper may write to the entire memory range. */ } - return update_stack_depth(env, state, min_off); + return 0; } static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, @@ -7192,6 +7249,12 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, } } +/* verify arguments to helpers or kfuncs consisting of a pointer and an access + * size. + * + * @regno is the register containing the access size. regno-1 is the register + * containing the pointer. + */ static int check_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, u32 regno, bool zero_size_allowed, @@ -7893,7 +7956,7 @@ static int process_iter_next_call(struct bpf_verifier_env *env, int insn_idx, /* switch to DRAINED state, but keep the depth unchanged */ /* mark current iter state as drained and assume returned NULL */ cur_iter->iter.state = BPF_ITER_STATE_DRAINED; - __mark_reg_const_zero(&cur_fr->regs[BPF_REG_0]); + __mark_reg_const_zero(env, &cur_fr->regs[BPF_REG_0]); return 0; } @@ -9396,7 +9459,7 @@ static int set_map_elem_callback_state(struct bpf_verifier_env *env, return err; callee->in_callback_fn = true; - callee->callback_ret_range = tnum_range(0, 1); + callee->callback_ret_range = retval_range(0, 1); return 0; } @@ -9418,7 +9481,7 @@ static int set_loop_callback_state(struct bpf_verifier_env *env, __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); callee->in_callback_fn = true; - callee->callback_ret_range = tnum_range(0, 1); + callee->callback_ret_range = retval_range(0, 1); return 0; } @@ -9448,7 +9511,7 @@ static int set_timer_callback_state(struct bpf_verifier_env *env, __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); callee->in_async_callback_fn = true; - callee->callback_ret_range = tnum_range(0, 1); + callee->callback_ret_range = retval_range(0, 1); return 0; } @@ -9476,7 +9539,7 @@ static int set_find_vma_callback_state(struct bpf_verifier_env *env, __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); callee->in_callback_fn = true; - callee->callback_ret_range = tnum_range(0, 1); + callee->callback_ret_range = retval_range(0, 1); return 0; } @@ -9499,7 +9562,7 @@ static int set_user_ringbuf_callback_state(struct bpf_verifier_env *env, __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); callee->in_callback_fn = true; - callee->callback_ret_range = tnum_range(0, 1); + callee->callback_ret_range = retval_range(0, 1); return 0; } @@ -9531,7 +9594,7 @@ static int set_rbtree_add_callback_state(struct bpf_verifier_env *env, __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); callee->in_callback_fn = true; - callee->callback_ret_range = tnum_range(0, 1); + callee->callback_ret_range = retval_range(0, 1); return 0; } @@ -9560,6 +9623,11 @@ static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env) return is_rbtree_lock_required_kfunc(kfunc_btf_id); } +static bool retval_range_within(struct bpf_retval_range range, const struct bpf_reg_state *reg) +{ + return range.minval <= reg->smin_value && reg->smax_value <= range.maxval; +} + static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) { struct bpf_verifier_state *state = env->cur_state, *prev_st; @@ -9583,15 +9651,21 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) caller = state->frame[state->curframe - 1]; if (callee->in_callback_fn) { - /* enforce R0 return value range [0, 1]. */ - struct tnum range = callee->callback_ret_range; - if (r0->type != SCALAR_VALUE) { verbose(env, "R0 not a scalar value\n"); return -EACCES; } - if (!tnum_in(range, r0->var_off)) { - verbose_invalid_scalar(env, r0, &range, "callback return", "R0"); + + /* we are going to rely on register's precise value */ + err = mark_reg_read(env, r0, r0->parent, REG_LIVE_READ64); + err = err ?: mark_chain_precision(env, BPF_REG_0); + if (err) + return err; + + /* enforce R0 return value range */ + if (!retval_range_within(callee->callback_ret_range, r0)) { + verbose_invalid_scalar(env, r0, callee->callback_ret_range, + "At callback return", "R0"); return -EINVAL; } if (!calls_callback(env, callee->callsite)) { @@ -11805,7 +11879,7 @@ static int fetch_kfunc_meta(struct bpf_verifier_env *env, return 0; } -static int check_return_code(struct bpf_verifier_env *env, int regno); +static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name); static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, int *insn_idx_p) @@ -11942,7 +12016,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, * to bpf_throw becomes the return value of the program. */ if (!env->exception_callback_subprog) { - err = check_return_code(env, BPF_REG_1); + err = check_return_code(env, BPF_REG_1, "R1"); if (err < 0) return err; } @@ -14972,12 +15046,13 @@ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) return 0; } -static int check_return_code(struct bpf_verifier_env *env, int regno) +static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name) { + const char *exit_ctx = "At program exit"; struct tnum enforce_attach_type_range = tnum_unknown; const struct bpf_prog *prog = env->prog; struct bpf_reg_state *reg; - struct tnum range = tnum_range(0, 1), const_0 = tnum_const(0); + struct bpf_retval_range range = retval_range(0, 1); enum bpf_prog_type prog_type = resolve_prog_type(env->prog); int err; struct bpf_func_state *frame = env->cur_state->frame[0]; @@ -15019,17 +15094,9 @@ static int check_return_code(struct bpf_verifier_env *env, int regno) if (frame->in_async_callback_fn) { /* enforce return zero from async callbacks like timer */ - if (reg->type != SCALAR_VALUE) { - verbose(env, "In async callback the register R%d is not a known value (%s)\n", - regno, reg_type_str(env, reg->type)); - return -EINVAL; - } - - if (!tnum_in(const_0, reg->var_off)) { - verbose_invalid_scalar(env, reg, &const_0, "async callback", "R0"); - return -EINVAL; - } - return 0; + exit_ctx = "At async callback return"; + range = retval_range(0, 0); + goto enforce_retval; } if (is_subprog && !frame->in_exception_callback_fn) { @@ -15052,14 +15119,14 @@ static int check_return_code(struct bpf_verifier_env *env, int regno) env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME || env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME || env->prog->expected_attach_type == BPF_CGROUP_UNIX_GETSOCKNAME) - range = tnum_range(1, 1); + range = retval_range(1, 1); if (env->prog->expected_attach_type == BPF_CGROUP_INET4_BIND || env->prog->expected_attach_type == BPF_CGROUP_INET6_BIND) - range = tnum_range(0, 3); + range = retval_range(0, 3); break; case BPF_PROG_TYPE_CGROUP_SKB: if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) { - range = tnum_range(0, 3); + range = retval_range(0, 3); enforce_attach_type_range = tnum_range(2, 3); } break; @@ -15072,13 +15139,13 @@ static int check_return_code(struct bpf_verifier_env *env, int regno) case BPF_PROG_TYPE_RAW_TRACEPOINT: if (!env->prog->aux->attach_btf_id) return 0; - range = tnum_const(0); + range = retval_range(0, 0); break; case BPF_PROG_TYPE_TRACING: switch (env->prog->expected_attach_type) { case BPF_TRACE_FENTRY: case BPF_TRACE_FEXIT: - range = tnum_const(0); + range = retval_range(0, 0); break; case BPF_TRACE_RAW_TP: case BPF_MODIFY_RETURN: @@ -15090,7 +15157,7 @@ static int check_return_code(struct bpf_verifier_env *env, int regno) } break; case BPF_PROG_TYPE_SK_LOOKUP: - range = tnum_range(SK_DROP, SK_PASS); + range = retval_range(SK_DROP, SK_PASS); break; case BPF_PROG_TYPE_LSM: @@ -15104,12 +15171,12 @@ static int check_return_code(struct bpf_verifier_env *env, int regno) /* Make sure programs that attach to void * hooks don't try to modify return value. */ - range = tnum_range(1, 1); + range = retval_range(1, 1); } break; case BPF_PROG_TYPE_NETFILTER: - range = tnum_range(NF_DROP, NF_ACCEPT); + range = retval_range(NF_DROP, NF_ACCEPT); break; case BPF_PROG_TYPE_EXT: /* freplace program can return anything as its return value @@ -15119,15 +15186,21 @@ static int check_return_code(struct bpf_verifier_env *env, int regno) return 0; } +enforce_retval: if (reg->type != SCALAR_VALUE) { - verbose(env, "At program exit the register R%d is not a known value (%s)\n", - regno, reg_type_str(env, reg->type)); + verbose(env, "%s the register R%d is not a known value (%s)\n", + exit_ctx, regno, reg_type_str(env, reg->type)); return -EINVAL; } - if (!tnum_in(range, reg->var_off)) { - verbose_invalid_scalar(env, reg, &range, "program exit", "R0"); - if (prog->expected_attach_type == BPF_LSM_CGROUP && + err = mark_chain_precision(env, regno); + if (err) + return err; + + if (!retval_range_within(range, reg)) { + verbose_invalid_scalar(env, reg, range, exit_ctx, reg_name); + if (!is_subprog && + prog->expected_attach_type == BPF_LSM_CGROUP && prog_type == BPF_PROG_TYPE_LSM && !prog->aux->attach_func_proto->type) verbose(env, "Note, BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n"); @@ -16892,7 +16965,8 @@ static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) * the precision needs to be propagated back in * the current state. */ - err = err ? : push_jmp_history(env, cur); + if (is_jmp_point(env, env->insn_idx)) + err = err ? : push_jmp_history(env, cur, 0); err = err ? : propagate_precision(env, &sl->state); if (err) return err; @@ -17117,6 +17191,9 @@ static int do_check(struct bpf_verifier_env *env) u8 class; int err; + /* reset current history entry on each new instruction */ + env->cur_hist_ent = NULL; + env->prev_insn_idx = prev_insn_idx; if (env->insn_idx >= insn_cnt) { verbose(env, "invalid insn idx %d insn_cnt %d\n", @@ -17156,7 +17233,7 @@ static int do_check(struct bpf_verifier_env *env) } if (is_jmp_point(env, env->insn_idx)) { - err = push_jmp_history(env, state); + err = push_jmp_history(env, state, 0); if (err) return err; } @@ -17410,7 +17487,7 @@ static int do_check(struct bpf_verifier_env *env) continue; } - err = check_return_code(env, BPF_REG_0); + err = check_return_code(env, BPF_REG_0, "R0"); if (err) return err; process_bpf_exit: @@ -17871,10 +17948,12 @@ static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env) return -E2BIG; } + if (env->prog->aux->sleepable) + atomic64_inc(&map->sleepable_refcnt); /* hold the map. If the program is rejected by verifier, * the map will be released by release_maps() or it * will be used by the valid program until it's unloaded - * and all maps are released in free_used_maps() + * and all maps are released in bpf_free_used_maps() */ bpf_map_inc(map); @@ -19091,9 +19170,7 @@ static int do_misc_fixups(struct bpf_verifier_env *env) env->exception_callback_subprog = env->subprog_cnt - 1; /* Don't update insn_cnt, as add_hidden_subprog always appends insns */ - env->subprog_info[env->exception_callback_subprog].is_cb = true; - env->subprog_info[env->exception_callback_subprog].is_async_cb = true; - env->subprog_info[env->exception_callback_subprog].is_exception_cb = true; + mark_subprog_exc_cb(env, env->exception_callback_subprog); } for (i = 0; i < insn_cnt; i++, insn++) { @@ -19793,7 +19870,7 @@ static void free_states(struct bpf_verifier_env *env) } } -static int do_check_common(struct bpf_verifier_env *env, int subprog, bool is_ex_cb) +static int do_check_common(struct bpf_verifier_env *env, int subprog) { bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); struct bpf_verifier_state *state; @@ -19824,9 +19901,23 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog, bool is_ex regs = state->frame[state->curframe]->regs; if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) { - ret = btf_prepare_func_args(env, subprog, regs, is_ex_cb); + u32 nargs; + + ret = btf_prepare_func_args(env, subprog, regs, &nargs); if (ret) goto out; + if (subprog_is_exc_cb(env, subprog)) { + state->frame[0]->in_exception_callback_fn = true; + /* We have already ensured that the callback returns an integer, just + * like all global subprogs. We need to determine it only has a single + * scalar argument. + */ + if (nargs != 1 || regs[BPF_REG_1].type != SCALAR_VALUE) { + verbose(env, "exception cb only supports single integer argument\n"); + ret = -EINVAL; + goto out; + } + } for (i = BPF_REG_1; i <= BPF_REG_5; i++) { if (regs[i].type == PTR_TO_CTX) mark_reg_known_zero(env, regs, i); @@ -19840,12 +19931,6 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog, bool is_ex regs[i].id = ++env->id_gen; } } - if (is_ex_cb) { - state->frame[0]->in_exception_callback_fn = true; - env->subprog_info[subprog].is_cb = true; - env->subprog_info[subprog].is_async_cb = true; - env->subprog_info[subprog].is_exception_cb = true; - } } else { /* 1st arg to a function */ regs[BPF_REG_1].type = PTR_TO_CTX; @@ -19925,7 +20010,7 @@ static int do_check_subprogs(struct bpf_verifier_env *env) env->insn_idx = env->subprog_info[i].start; WARN_ON_ONCE(env->insn_idx == 0); - ret = do_check_common(env, i, env->exception_callback_subprog == i); + ret = do_check_common(env, i); if (ret) { return ret; } else if (env->log.level & BPF_LOG_LEVEL) { @@ -19955,7 +20040,7 @@ static int do_check_main(struct bpf_verifier_env *env) int ret; env->insn_idx = 0; - ret = do_check_common(env, 0, false); + ret = do_check_common(env, 0); if (!ret) env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; return ret; @@ -20509,7 +20594,12 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3 env->prog = *prog; env->ops = bpf_verifier_ops[env->prog->type]; env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel); - is_priv = bpf_capable(); + + env->allow_ptr_leaks = bpf_allow_ptr_leaks(env->prog->aux->token); + env->allow_uninit_stack = bpf_allow_uninit_stack(env->prog->aux->token); + env->bypass_spec_v1 = bpf_bypass_spec_v1(env->prog->aux->token); + env->bypass_spec_v4 = bpf_bypass_spec_v4(env->prog->aux->token); + env->bpf_capable = is_priv = bpf_token_capable(env->prog->aux->token, CAP_BPF); bpf_get_btf_vmlinux(); @@ -20541,12 +20631,6 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3 if (attr->prog_flags & BPF_F_ANY_ALIGNMENT) env->strict_alignment = false; - env->allow_ptr_leaks = bpf_allow_ptr_leaks(); - env->allow_uninit_stack = bpf_allow_uninit_stack(); - env->bypass_spec_v1 = bpf_bypass_spec_v1(); - env->bypass_spec_v4 = bpf_bypass_spec_v4(); - env->bpf_capable = bpf_capable(); - if (is_priv) env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; env->test_reg_invariants = attr->prog_flags & BPF_F_TEST_REG_INVARIANTS; diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index c284a4ad0315cf..492d60e9c48000 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -24,6 +24,7 @@ #include #include #include +#include #include @@ -41,6 +42,9 @@ #define bpf_event_rcu_dereference(p) \ rcu_dereference_protected(p, lockdep_is_held(&bpf_event_mutex)) +#define MAX_UPROBE_MULTI_CNT (1U << 20) +#define MAX_KPROBE_MULTI_CNT (1U << 20) + #ifdef CONFIG_MODULES struct bpf_trace_module { struct module *module; @@ -1431,6 +1435,72 @@ static int __init bpf_key_sig_kfuncs_init(void) late_initcall(bpf_key_sig_kfuncs_init); #endif /* CONFIG_KEYS */ +/* filesystem kfuncs */ +__bpf_kfunc_start_defs(); + +/** + * bpf_get_file_xattr - get xattr of a file + * @file: file to get xattr from + * @name__str: name of the xattr + * @value_ptr: output buffer of the xattr value + * + * Get xattr *name__str* of *file* and store the output in *value_ptr*. + * + * For security reasons, only *name__str* with prefix "user." is allowed. + * + * Return: 0 on success, a negative value on error. + */ +__bpf_kfunc int bpf_get_file_xattr(struct file *file, const char *name__str, + struct bpf_dynptr_kern *value_ptr) +{ + struct dentry *dentry; + u32 value_len; + void *value; + int ret; + + if (strncmp(name__str, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN)) + return -EPERM; + + value_len = __bpf_dynptr_size(value_ptr); + value = __bpf_dynptr_data_rw(value_ptr, value_len); + if (!value) + return -EINVAL; + + dentry = file_dentry(file); + ret = inode_permission(&nop_mnt_idmap, dentry->d_inode, MAY_READ); + if (ret) + return ret; + return __vfs_getxattr(dentry, dentry->d_inode, name__str, value, value_len); +} + +__bpf_kfunc_end_defs(); + +BTF_SET8_START(fs_kfunc_set_ids) +BTF_ID_FLAGS(func, bpf_get_file_xattr, KF_SLEEPABLE | KF_TRUSTED_ARGS) +BTF_SET8_END(fs_kfunc_set_ids) + +static int bpf_get_file_xattr_filter(const struct bpf_prog *prog, u32 kfunc_id) +{ + if (!btf_id_set8_contains(&fs_kfunc_set_ids, kfunc_id)) + return 0; + + /* Only allow to attach from LSM hooks, to avoid recursion */ + return prog->type != BPF_PROG_TYPE_LSM ? -EACCES : 0; +} + +static const struct btf_kfunc_id_set bpf_fs_kfunc_set = { + .owner = THIS_MODULE, + .set = &fs_kfunc_set_ids, + .filter = bpf_get_file_xattr_filter, +}; + +static int __init bpf_fs_kfuncs_init(void) +{ + return register_btf_kfunc_id_set(BPF_PROG_TYPE_LSM, &bpf_fs_kfunc_set); +} + +late_initcall(bpf_fs_kfuncs_init); + static const struct bpf_func_proto * bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { @@ -1559,7 +1629,7 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_trace_vprintk: return bpf_get_trace_vprintk_proto(); default: - return bpf_base_func_proto(func_id); + return bpf_base_func_proto(func_id, prog); } } @@ -2903,6 +2973,8 @@ int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr cnt = attr->link_create.kprobe_multi.cnt; if (!cnt) return -EINVAL; + if (cnt > MAX_KPROBE_MULTI_CNT) + return -E2BIG; size = cnt * sizeof(*addrs); addrs = kvmalloc_array(cnt, sizeof(*addrs), GFP_KERNEL); @@ -3277,6 +3349,8 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr if (!upath || !uoffsets || !cnt) return -EINVAL; + if (cnt > MAX_UPROBE_MULTI_CNT) + return -E2BIG; uref_ctr_offsets = u64_to_user_ptr(attr->link_create.uprobe_multi.ref_ctr_offsets); ucookies = u64_to_user_ptr(attr->link_create.uprobe_multi.cookies); @@ -3317,15 +3391,19 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr goto error_free; for (i = 0; i < cnt; i++) { - if (ucookies && __get_user(uprobes[i].cookie, ucookies + i)) { + if (__get_user(uprobes[i].offset, uoffsets + i)) { err = -EFAULT; goto error_free; } + if (uprobes[i].offset < 0) { + err = -EINVAL; + goto error_free; + } if (uref_ctr_offsets && __get_user(uprobes[i].ref_ctr_offset, uref_ctr_offsets + i)) { err = -EFAULT; goto error_free; } - if (__get_user(uprobes[i].offset, uoffsets + i)) { + if (ucookies && __get_user(uprobes[i].cookie, ucookies + i)) { err = -EFAULT; goto error_free; } diff --git a/lib/test_bpf.c b/lib/test_bpf.c index e380fdf756dbe0..569e6d2dc55c72 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -6277,7 +6277,7 @@ static struct bpf_test tests[] = { }, /* BPF_ALU64 | BPF_MOD | BPF_K off=1 (SMOD64) */ { - "ALU64_SMOD_X: -7 % 2 = -1", + "ALU64_SMOD_K: -7 % 2 = -1", .u.insns_int = { BPF_LD_IMM64(R0, -7), BPF_ALU64_IMM_OFF(BPF_MOD, R0, 2, 1), diff --git a/net/bpf/bpf_dummy_struct_ops.c b/net/bpf/bpf_dummy_struct_ops.c index 5918d1b32e1960..8906f7bdf4a92a 100644 --- a/net/bpf/bpf_dummy_struct_ops.c +++ b/net/bpf/bpf_dummy_struct_ops.c @@ -12,6 +12,11 @@ extern struct bpf_struct_ops bpf_bpf_dummy_ops; /* A common type for test_N with return value in bpf_dummy_ops */ typedef int (*dummy_ops_test_ret_fn)(struct bpf_dummy_ops_state *state, ...); +static int dummy_ops_test_ret_function(struct bpf_dummy_ops_state *state, ...) +{ + return 0; +} + struct bpf_dummy_ops_test_args { u64 args[MAX_BPF_FUNC_ARGS]; struct bpf_dummy_ops_state state; @@ -62,7 +67,7 @@ static int dummy_ops_copy_args(struct bpf_dummy_ops_test_args *args) static int dummy_ops_call_op(void *image, struct bpf_dummy_ops_test_args *args) { - dummy_ops_test_ret_fn test = (void *)image; + dummy_ops_test_ret_fn test = (void *)image + cfi_get_offset(); struct bpf_dummy_ops_state *state = NULL; /* state needs to be NULL if args[0] is 0 */ @@ -101,12 +106,11 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, goto out; } - image = bpf_jit_alloc_exec(PAGE_SIZE); + image = arch_alloc_bpf_trampoline(PAGE_SIZE); if (!image) { err = -ENOMEM; goto out; } - set_vm_flush_reset_perms(image); link = kzalloc(sizeof(*link), GFP_USER); if (!link) { @@ -120,11 +124,12 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, op_idx = prog->expected_attach_type; err = bpf_struct_ops_prepare_trampoline(tlinks, link, &st_ops->func_models[op_idx], + &dummy_ops_test_ret_function, image, image + PAGE_SIZE); if (err < 0) goto out; - set_memory_rox((long)image, 1); + arch_protect_bpf_trampoline(image, PAGE_SIZE); prog_ret = dummy_ops_call_op(image, args); err = dummy_ops_copy_args(args); @@ -134,7 +139,7 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, err = -EFAULT; out: kfree(args); - bpf_jit_free_exec(image); + arch_free_bpf_trampoline(image, PAGE_SIZE); if (link) bpf_link_put(&link->link); kfree(tlinks); @@ -220,6 +225,28 @@ static void bpf_dummy_unreg(void *kdata) { } +static int bpf_dummy_test_1(struct bpf_dummy_ops_state *cb) +{ + return 0; +} + +static int bpf_dummy_test_2(struct bpf_dummy_ops_state *cb, int a1, unsigned short a2, + char a3, unsigned long a4) +{ + return 0; +} + +static int bpf_dummy_test_sleepable(struct bpf_dummy_ops_state *cb) +{ + return 0; +} + +static struct bpf_dummy_ops __bpf_bpf_dummy_ops = { + .test_1 = bpf_dummy_test_1, + .test_2 = bpf_dummy_test_2, + .test_sleepable = bpf_dummy_test_sleepable, +}; + struct bpf_struct_ops bpf_bpf_dummy_ops = { .verifier_ops = &bpf_dummy_verifier_ops, .init = bpf_dummy_init, @@ -228,4 +255,5 @@ struct bpf_struct_ops bpf_bpf_dummy_ops = { .reg = bpf_dummy_reg, .unreg = bpf_dummy_unreg, .name = "bpf_dummy_ops", + .cfi_stubs = &__bpf_bpf_dummy_ops, }; diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c index 711cf5d59816b9..dfd91937401783 100644 --- a/net/bpf/test_run.c +++ b/net/bpf/test_run.c @@ -600,10 +600,21 @@ __bpf_kfunc void bpf_kfunc_call_test_release(struct prog_test_ref_kfunc *p) refcount_dec(&p->cnt); } +__bpf_kfunc void bpf_kfunc_call_test_release_dtor(void *p) +{ + bpf_kfunc_call_test_release(p); +} +CFI_NOSEAL(bpf_kfunc_call_test_release_dtor); + __bpf_kfunc void bpf_kfunc_call_memb_release(struct prog_test_member *p) { } +__bpf_kfunc void bpf_kfunc_call_memb_release_dtor(void *p) +{ +} +CFI_NOSEAL(bpf_kfunc_call_memb_release_dtor); + __bpf_kfunc_end_defs(); BTF_SET8_START(bpf_test_modify_return_ids) @@ -1671,9 +1682,9 @@ static const struct btf_kfunc_id_set bpf_prog_test_kfunc_set = { BTF_ID_LIST(bpf_prog_test_dtor_kfunc_ids) BTF_ID(struct, prog_test_ref_kfunc) -BTF_ID(func, bpf_kfunc_call_test_release) +BTF_ID(func, bpf_kfunc_call_test_release_dtor) BTF_ID(struct, prog_test_member) -BTF_ID(func, bpf_kfunc_call_memb_release) +BTF_ID(func, bpf_kfunc_call_memb_release_dtor) static int __init bpf_prog_test_run_init(void) { diff --git a/net/core/bpf_sk_storage.c b/net/core/bpf_sk_storage.c index cca7594be92ec6..6c4d90b24d467e 100644 --- a/net/core/bpf_sk_storage.c +++ b/net/core/bpf_sk_storage.c @@ -275,9 +275,10 @@ BPF_CALL_2(bpf_sk_storage_delete, struct bpf_map *, map, struct sock *, sk) static int bpf_sk_storage_charge(struct bpf_local_storage_map *smap, void *owner, u32 size) { - int optmem_max = READ_ONCE(sysctl_optmem_max); struct sock *sk = (struct sock *)owner; + int optmem_max; + optmem_max = READ_ONCE(sock_net(sk)->core.sysctl_optmem_max); /* same check as in sock_kmalloc() */ if (size <= optmem_max && atomic_read(&sk->sk_omem_alloc) + size < optmem_max) { diff --git a/net/core/filter.c b/net/core/filter.c index eedb33f3e9982f..4ff6100c6a2733 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -87,7 +87,7 @@ #include "dev.h" static const struct bpf_func_proto * -bpf_sk_base_func_proto(enum bpf_func_id func_id); +bpf_sk_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog); int copy_bpf_fprog_from_user(struct sock_fprog *dst, sockptr_t src, int len) { @@ -1219,8 +1219,8 @@ void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp) */ static bool __sk_filter_charge(struct sock *sk, struct sk_filter *fp) { + int optmem_max = READ_ONCE(sock_net(sk)->core.sysctl_optmem_max); u32 filter_size = bpf_prog_size(fp->prog->len); - int optmem_max = READ_ONCE(sysctl_optmem_max); /* same check as in sock_kmalloc() */ if (filter_size <= optmem_max && @@ -1550,12 +1550,13 @@ EXPORT_SYMBOL_GPL(sk_attach_filter); int sk_reuseport_attach_filter(struct sock_fprog *fprog, struct sock *sk) { struct bpf_prog *prog = __get_filter(fprog, sk); - int err; + int err, optmem_max; if (IS_ERR(prog)) return PTR_ERR(prog); - if (bpf_prog_size(prog->len) > READ_ONCE(sysctl_optmem_max)) + optmem_max = READ_ONCE(sock_net(sk)->core.sysctl_optmem_max); + if (bpf_prog_size(prog->len) > optmem_max) err = -ENOMEM; else err = reuseport_attach_prog(sk, prog); @@ -1594,7 +1595,7 @@ int sk_attach_bpf(u32 ufd, struct sock *sk) int sk_reuseport_attach_bpf(u32 ufd, struct sock *sk) { struct bpf_prog *prog; - int err; + int err, optmem_max; if (sock_flag(sk, SOCK_FILTER_LOCKED)) return -EPERM; @@ -1622,7 +1623,8 @@ int sk_reuseport_attach_bpf(u32 ufd, struct sock *sk) } } else { /* BPF_PROG_TYPE_SOCKET_FILTER */ - if (bpf_prog_size(prog->len) > READ_ONCE(sysctl_optmem_max)) { + optmem_max = READ_ONCE(sock_net(sk)->core.sysctl_optmem_max); + if (bpf_prog_size(prog->len) > optmem_max) { err = -ENOMEM; goto err_prog_put; } @@ -7860,7 +7862,7 @@ sock_filter_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_ktime_get_coarse_ns: return &bpf_ktime_get_coarse_ns_proto; default: - return bpf_base_func_proto(func_id); + return bpf_base_func_proto(func_id, prog); } } @@ -7953,7 +7955,7 @@ sock_addr_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) return NULL; } default: - return bpf_sk_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id, prog); } } @@ -7972,7 +7974,7 @@ sk_filter_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_perf_event_output: return &bpf_skb_event_output_proto; default: - return bpf_sk_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id, prog); } } @@ -8159,7 +8161,7 @@ tc_cls_act_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) #endif #endif default: - return bpf_sk_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id, prog); } } @@ -8218,7 +8220,7 @@ xdp_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) #endif #endif default: - return bpf_sk_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id, prog); } #if IS_MODULE(CONFIG_NF_CONNTRACK) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF_MODULES) @@ -8279,7 +8281,7 @@ sock_ops_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) return &bpf_tcp_sock_proto; #endif /* CONFIG_INET */ default: - return bpf_sk_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id, prog); } } @@ -8321,7 +8323,7 @@ sk_msg_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) return &bpf_get_cgroup_classid_curr_proto; #endif default: - return bpf_sk_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id, prog); } } @@ -8365,7 +8367,7 @@ sk_skb_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) return &bpf_skc_lookup_tcp_proto; #endif default: - return bpf_sk_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id, prog); } } @@ -8376,7 +8378,7 @@ flow_dissector_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_skb_load_bytes: return &bpf_flow_dissector_load_bytes_proto; default: - return bpf_sk_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id, prog); } } @@ -8403,7 +8405,7 @@ lwt_out_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_skb_under_cgroup: return &bpf_skb_under_cgroup_proto; default: - return bpf_sk_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id, prog); } } @@ -8578,7 +8580,7 @@ static bool cg_skb_is_valid_access(int off, int size, return false; case bpf_ctx_range(struct __sk_buff, data): case bpf_ctx_range(struct __sk_buff, data_end): - if (!bpf_capable()) + if (!bpf_token_capable(prog->aux->token, CAP_BPF)) return false; break; } @@ -8590,7 +8592,7 @@ static bool cg_skb_is_valid_access(int off, int size, case bpf_ctx_range_till(struct __sk_buff, cb[0], cb[4]): break; case bpf_ctx_range(struct __sk_buff, tstamp): - if (!bpf_capable()) + if (!bpf_token_capable(prog->aux->token, CAP_BPF)) return false; break; default: @@ -11234,7 +11236,7 @@ sk_reuseport_func_proto(enum bpf_func_id func_id, case BPF_FUNC_ktime_get_coarse_ns: return &bpf_ktime_get_coarse_ns_proto; default: - return bpf_base_func_proto(func_id); + return bpf_base_func_proto(func_id, prog); } } @@ -11416,7 +11418,7 @@ sk_lookup_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_sk_release: return &bpf_sk_release_proto; default: - return bpf_sk_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id, prog); } } @@ -11750,7 +11752,7 @@ const struct bpf_func_proto bpf_sock_from_file_proto = { }; static const struct bpf_func_proto * -bpf_sk_base_func_proto(enum bpf_func_id func_id) +bpf_sk_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { const struct bpf_func_proto *func; @@ -11779,10 +11781,10 @@ bpf_sk_base_func_proto(enum bpf_func_id func_id) case BPF_FUNC_ktime_get_coarse_ns: return &bpf_ktime_get_coarse_ns_proto; default: - return bpf_base_func_proto(func_id); + return bpf_base_func_proto(func_id, prog); } - if (!perfmon_capable()) + if (!bpf_token_capable(prog->aux->token, CAP_PERFMON)) return NULL; return func; diff --git a/net/core/net_namespace.c b/net/core/net_namespace.c index cb8bcbff9e83a1..72799533426b61 100644 --- a/net/core/net_namespace.c +++ b/net/core/net_namespace.c @@ -372,6 +372,10 @@ static __net_init int setup_net(struct net *net, struct user_namespace *user_ns) static int __net_init net_defaults_init_net(struct net *net) { net->core.sysctl_somaxconn = SOMAXCONN; + /* Limits per socket sk_omem_alloc usage. + * TCP zerocopy regular usage needs 128 KB. + */ + net->core.sysctl_optmem_max = 128 * 1024; net->core.sysctl_txrehash = SOCK_TXREHASH_ENABLED; return 0; diff --git a/net/core/page_pool.c b/net/core/page_pool.c index dd5a72533f2b9d..4933762e5a6ba4 100644 --- a/net/core/page_pool.c +++ b/net/core/page_pool.c @@ -28,7 +28,7 @@ #define DEFER_TIME (msecs_to_jiffies(1000)) #define DEFER_WARN_INTERVAL (60 * HZ) -#define BIAS_MAX LONG_MAX +#define BIAS_MAX (LONG_MAX >> 1) #ifdef CONFIG_PAGE_POOL_STATS /* alloc_stat_inc is intended to be used in softirq context */ diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c index 5e0ab4c08f72ed..94c4572512b819 100644 --- a/net/core/rtnetlink.c +++ b/net/core/rtnetlink.c @@ -342,8 +342,7 @@ int rtnl_unregister(int protocol, int msgtype) return -ENOENT; } - link = rtnl_dereference(tab[msgindex]); - RCU_INIT_POINTER(tab[msgindex], NULL); + link = rcu_replace_pointer_rtnl(tab[msgindex], NULL); rtnl_unlock(); kfree_rcu(link, rcu); @@ -368,18 +367,13 @@ void rtnl_unregister_all(int protocol) BUG_ON(protocol < 0 || protocol > RTNL_FAMILY_MAX); rtnl_lock(); - tab = rtnl_dereference(rtnl_msg_handlers[protocol]); + tab = rcu_replace_pointer_rtnl(rtnl_msg_handlers[protocol], NULL); if (!tab) { rtnl_unlock(); return; } - RCU_INIT_POINTER(rtnl_msg_handlers[protocol], NULL); for (msgindex = 0; msgindex < RTM_NR_MSGTYPES; msgindex++) { - link = rtnl_dereference(tab[msgindex]); - if (!link) - continue; - - RCU_INIT_POINTER(tab[msgindex], NULL); + link = rcu_replace_pointer_rtnl(tab[msgindex], NULL); kfree_rcu(link, rcu); } rtnl_unlock(); diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 83af8aaeb893b1..4d4b11b0a83deb 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -890,6 +890,11 @@ static void skb_clone_fraglist(struct sk_buff *skb) skb_get(list); } +static bool is_pp_page(struct page *page) +{ + return (page->pp_magic & ~0x3UL) == PP_SIGNATURE; +} + #if IS_ENABLED(CONFIG_PAGE_POOL) bool napi_pp_put_page(struct page *page, bool napi_safe) { @@ -905,7 +910,7 @@ bool napi_pp_put_page(struct page *page, bool napi_safe) * and page_is_pfmemalloc() is checked in __page_pool_put_page() * to avoid recycling the pfmemalloc page. */ - if (unlikely((page->pp_magic & ~0x3UL) != PP_SIGNATURE)) + if (unlikely(!is_pp_page(page))) return false; pp = page->pp; @@ -942,6 +947,37 @@ static bool skb_pp_recycle(struct sk_buff *skb, void *data, bool napi_safe) return napi_pp_put_page(virt_to_page(data), napi_safe); } +/** + * skb_pp_frag_ref() - Increase fragment references of a page pool aware skb + * @skb: page pool aware skb + * + * Increase the fragment reference count (pp_ref_count) of a skb. This is + * intended to gain fragment references only for page pool aware skbs, + * i.e. when skb->pp_recycle is true, and not for fragments in a + * non-pp-recycling skb. It has a fallback to increase references on normal + * pages, as page pool aware skbs may also have normal page fragments. + */ +static int skb_pp_frag_ref(struct sk_buff *skb) +{ + struct skb_shared_info *shinfo; + struct page *head_page; + int i; + + if (!skb->pp_recycle) + return -EINVAL; + + shinfo = skb_shinfo(skb); + + for (i = 0; i < shinfo->nr_frags; i++) { + head_page = compound_head(skb_frag_page(&shinfo->frags[i])); + if (likely(is_pp_page(head_page))) + page_pool_ref_page(head_page); + else + page_ref_inc(head_page); + } + return 0; +} + static void skb_kfree_head(void *head, unsigned int end_offset) { if (end_offset == SKB_SMALL_HEAD_HEADROOM) @@ -5765,17 +5801,12 @@ bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from, return false; /* In general, avoid mixing page_pool and non-page_pool allocated - * pages within the same SKB. Additionally avoid dealing with clones - * with page_pool pages, in case the SKB is using page_pool fragment - * references (page_pool_alloc_frag()). Since we only take full page - * references for cloned SKBs at the moment that would result in - * inconsistent reference counts. - * In theory we could take full references if @from is cloned and - * !@to->pp_recycle but its tricky (due to potential race with - * the clone disappearing) and rare, so not worth dealing with. + * pages within the same SKB. In theory we could take full + * references if @from is cloned and !@to->pp_recycle but its + * tricky (due to potential race with the clone disappearing) and + * rare, so not worth dealing with. */ - if (to->pp_recycle != from->pp_recycle || - (from->pp_recycle && skb_cloned(from))) + if (to->pp_recycle != from->pp_recycle) return false; if (len <= skb_tailroom(to)) { @@ -5832,8 +5863,10 @@ bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from, /* if the skb is not cloned this does nothing * since we set nr_frags to 0. */ - for (i = 0; i < from_shinfo->nr_frags; i++) - __skb_frag_ref(&from_shinfo->frags[i]); + if (skb_pp_frag_ref(from)) { + for (i = 0; i < from_shinfo->nr_frags; i++) + __skb_frag_ref(&from_shinfo->frags[i]); + } to->truesize += delta; to->len += len; diff --git a/net/core/sock.c b/net/core/sock.c index fef349dd72fa73..446e945f736b3e 100644 --- a/net/core/sock.c +++ b/net/core/sock.c @@ -283,10 +283,6 @@ EXPORT_SYMBOL(sysctl_rmem_max); __u32 sysctl_wmem_default __read_mostly = SK_WMEM_MAX; __u32 sysctl_rmem_default __read_mostly = SK_RMEM_MAX; -/* Maximal space eaten by iovec or ancillary data plus some space */ -int sysctl_optmem_max __read_mostly = sizeof(unsigned long)*(2*UIO_MAXIOV+512); -EXPORT_SYMBOL(sysctl_optmem_max); - int sysctl_tstamp_allow_data __read_mostly = 1; DEFINE_STATIC_KEY_FALSE(memalloc_socks_key); @@ -2651,7 +2647,7 @@ struct sk_buff *sock_omalloc(struct sock *sk, unsigned long size, /* small safe race: SKB_TRUESIZE may differ from final skb->truesize */ if (atomic_read(&sk->sk_omem_alloc) + SKB_TRUESIZE(size) > - READ_ONCE(sysctl_optmem_max)) + READ_ONCE(sock_net(sk)->core.sysctl_optmem_max)) return NULL; skb = alloc_skb(size, priority); @@ -2669,7 +2665,7 @@ struct sk_buff *sock_omalloc(struct sock *sk, unsigned long size, */ void *sock_kmalloc(struct sock *sk, int size, gfp_t priority) { - int optmem_max = READ_ONCE(sysctl_optmem_max); + int optmem_max = READ_ONCE(sock_net(sk)->core.sysctl_optmem_max); if ((unsigned int)size <= optmem_max && atomic_read(&sk->sk_omem_alloc) + size < optmem_max) { diff --git a/net/core/sysctl_net_core.c b/net/core/sysctl_net_core.c index 03f1edb948d7df..0f0cb1465e0892 100644 --- a/net/core/sysctl_net_core.c +++ b/net/core/sysctl_net_core.c @@ -508,13 +508,6 @@ static struct ctl_table net_core_table[] = { .mode = 0644, .proc_handler = proc_dointvec, }, - { - .procname = "optmem_max", - .data = &sysctl_optmem_max, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec - }, { .procname = "tstamp_allow_data", .data = &sysctl_tstamp_allow_data, @@ -673,6 +666,14 @@ static struct ctl_table netns_core_table[] = { .extra1 = SYSCTL_ZERO, .proc_handler = proc_dointvec_minmax }, + { + .procname = "optmem_max", + .data = &init_net.core.sysctl_optmem_max, + .maxlen = sizeof(int), + .mode = 0644, + .extra1 = SYSCTL_ZERO, + .proc_handler = proc_dointvec_minmax + }, { .procname = "txrehash", .data = &init_net.core.sysctl_txrehash, diff --git a/net/core/xdp.c b/net/core/xdp.c index b6f1d6dab3f2e3..4869c1c2d8f3d9 100644 --- a/net/core/xdp.c +++ b/net/core/xdp.c @@ -736,6 +736,39 @@ __bpf_kfunc int bpf_xdp_metadata_rx_hash(const struct xdp_md *ctx, u32 *hash, return -EOPNOTSUPP; } +/** + * bpf_xdp_metadata_rx_vlan_tag - Get XDP packet outermost VLAN tag + * @ctx: XDP context pointer. + * @vlan_proto: Destination pointer for VLAN Tag protocol identifier (TPID). + * @vlan_tci: Destination pointer for VLAN TCI (VID + DEI + PCP) + * + * In case of success, ``vlan_proto`` contains *Tag protocol identifier (TPID)*, + * usually ``ETH_P_8021Q`` or ``ETH_P_8021AD``, but some networks can use + * custom TPIDs. ``vlan_proto`` is stored in **network byte order (BE)** + * and should be used as follows: + * ``if (vlan_proto == bpf_htons(ETH_P_8021Q)) do_something();`` + * + * ``vlan_tci`` contains the remaining 16 bits of a VLAN tag. + * Driver is expected to provide those in **host byte order (usually LE)**, + * so the bpf program should not perform byte conversion. + * According to 802.1Q standard, *VLAN TCI (Tag control information)* + * is a bit field that contains: + * *VLAN identifier (VID)* that can be read with ``vlan_tci & 0xfff``, + * *Drop eligible indicator (DEI)* - 1 bit, + * *Priority code point (PCP)* - 3 bits. + * For detailed meaning of DEI and PCP, please refer to other sources. + * + * Return: + * * Returns 0 on success or ``-errno`` on error. + * * ``-EOPNOTSUPP`` : device driver doesn't implement kfunc + * * ``-ENODATA`` : VLAN tag was not stripped or is not available + */ +__bpf_kfunc int bpf_xdp_metadata_rx_vlan_tag(const struct xdp_md *ctx, + __be16 *vlan_proto, u16 *vlan_tci) +{ + return -EOPNOTSUPP; +} + __bpf_kfunc_end_defs(); BTF_SET8_START(xdp_metadata_kfunc_ids) diff --git a/net/ipv4/bpf_tcp_ca.c b/net/ipv4/bpf_tcp_ca.c index 39dcccf0f174b9..634cfafa583dc9 100644 --- a/net/ipv4/bpf_tcp_ca.c +++ b/net/ipv4/bpf_tcp_ca.c @@ -191,7 +191,7 @@ bpf_tcp_ca_get_func_proto(enum bpf_func_id func_id, case BPF_FUNC_ktime_get_coarse_ns: return &bpf_ktime_get_coarse_ns_proto; default: - return bpf_base_func_proto(func_id); + return bpf_base_func_proto(func_id, prog); } } @@ -271,6 +271,74 @@ static int bpf_tcp_ca_validate(void *kdata) return tcp_validate_congestion_control(kdata); } +static u32 bpf_tcp_ca_ssthresh(struct sock *sk) +{ + return 0; +} + +static void bpf_tcp_ca_cong_avoid(struct sock *sk, u32 ack, u32 acked) +{ +} + +static void bpf_tcp_ca_set_state(struct sock *sk, u8 new_state) +{ +} + +static void bpf_tcp_ca_cwnd_event(struct sock *sk, enum tcp_ca_event ev) +{ +} + +static void bpf_tcp_ca_in_ack_event(struct sock *sk, u32 flags) +{ +} + +static void bpf_tcp_ca_pkts_acked(struct sock *sk, const struct ack_sample *sample) +{ +} + +static u32 bpf_tcp_ca_min_tso_segs(struct sock *sk) +{ + return 0; +} + +static void bpf_tcp_ca_cong_control(struct sock *sk, const struct rate_sample *rs) +{ +} + +static u32 bpf_tcp_ca_undo_cwnd(struct sock *sk) +{ + return 0; +} + +static u32 bpf_tcp_ca_sndbuf_expand(struct sock *sk) +{ + return 0; +} + +static void __bpf_tcp_ca_init(struct sock *sk) +{ +} + +static void __bpf_tcp_ca_release(struct sock *sk) +{ +} + +static struct tcp_congestion_ops __bpf_ops_tcp_congestion_ops = { + .ssthresh = bpf_tcp_ca_ssthresh, + .cong_avoid = bpf_tcp_ca_cong_avoid, + .set_state = bpf_tcp_ca_set_state, + .cwnd_event = bpf_tcp_ca_cwnd_event, + .in_ack_event = bpf_tcp_ca_in_ack_event, + .pkts_acked = bpf_tcp_ca_pkts_acked, + .min_tso_segs = bpf_tcp_ca_min_tso_segs, + .cong_control = bpf_tcp_ca_cong_control, + .undo_cwnd = bpf_tcp_ca_undo_cwnd, + .sndbuf_expand = bpf_tcp_ca_sndbuf_expand, + + .init = __bpf_tcp_ca_init, + .release = __bpf_tcp_ca_release, +}; + struct bpf_struct_ops bpf_tcp_congestion_ops = { .verifier_ops = &bpf_tcp_ca_verifier_ops, .reg = bpf_tcp_ca_reg, @@ -281,6 +349,7 @@ struct bpf_struct_ops bpf_tcp_congestion_ops = { .init = bpf_tcp_ca_init, .validate = bpf_tcp_ca_validate, .name = "tcp_congestion_ops", + .cfi_stubs = &__bpf_ops_tcp_congestion_ops, }; static int __init bpf_tcp_ca_kfunc_init(void) diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c index 70be0f6fe879ea..bd325b029dd12c 100644 --- a/net/ipv4/inet_connection_sock.c +++ b/net/ipv4/inet_connection_sock.c @@ -117,16 +117,25 @@ bool inet_rcv_saddr_any(const struct sock *sk) return !sk->sk_rcv_saddr; } -void inet_sk_get_local_port_range(const struct sock *sk, int *low, int *high) +/** + * inet_sk_get_local_port_range - fetch ephemeral ports range + * @sk: socket + * @low: pointer to low port + * @high: pointer to high port + * + * Fetch netns port range (/proc/sys/net/ipv4/ip_local_port_range) + * Range can be overridden if socket got IP_LOCAL_PORT_RANGE option. + * Returns true if IP_LOCAL_PORT_RANGE was set on this socket. + */ +bool inet_sk_get_local_port_range(const struct sock *sk, int *low, int *high) { - const struct inet_sock *inet = inet_sk(sk); - const struct net *net = sock_net(sk); int lo, hi, sk_lo, sk_hi; + bool local_range = false; u32 sk_range; - inet_get_local_port_range(net, &lo, &hi); + inet_get_local_port_range(sock_net(sk), &lo, &hi); - sk_range = READ_ONCE(inet->local_port_range); + sk_range = READ_ONCE(inet_sk(sk)->local_port_range); if (unlikely(sk_range)) { sk_lo = sk_range & 0xffff; sk_hi = sk_range >> 16; @@ -135,10 +144,12 @@ void inet_sk_get_local_port_range(const struct sock *sk, int *low, int *high) lo = sk_lo; if (lo <= sk_hi && sk_hi <= hi) hi = sk_hi; + local_range = true; } *low = lo; *high = hi; + return local_range; } EXPORT_SYMBOL(inet_sk_get_local_port_range); diff --git a/net/ipv4/inet_hashtables.c b/net/ipv4/inet_hashtables.c index a532f749e47781..9ff201bc4e6d2d 100644 --- a/net/ipv4/inet_hashtables.c +++ b/net/ipv4/inet_hashtables.c @@ -1012,7 +1012,8 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row, bool tb_created = false; u32 remaining, offset; int ret, i, low, high; - int l3mdev; + bool local_ports; + int step, l3mdev; u32 index; if (port) { @@ -1024,10 +1025,12 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row, l3mdev = inet_sk_bound_l3mdev(sk); - inet_sk_get_local_port_range(sk, &low, &high); + local_ports = inet_sk_get_local_port_range(sk, &low, &high); + step = local_ports ? 1 : 2; + high++; /* [32768, 60999] -> [32768, 61000[ */ remaining = high - low; - if (likely(remaining > 1)) + if (!local_ports && remaining > 1) remaining &= ~1U; get_random_sleepable_once(table_perturb, @@ -1040,10 +1043,11 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row, /* In first pass we try ports of @low parity. * inet_csk_get_port() does the opposite choice. */ - offset &= ~1U; + if (!local_ports) + offset &= ~1U; other_parity_scan: port = low + offset; - for (i = 0; i < remaining; i += 2, port += 2) { + for (i = 0; i < remaining; i += step, port += step) { if (unlikely(port >= high)) port -= remaining; if (inet_is_local_reserved_port(net, port)) @@ -1083,10 +1087,11 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row, cond_resched(); } - offset++; - if ((offset & 1) && remaining > 1) - goto other_parity_scan; - + if (!local_ports) { + offset++; + if ((offset & 1) && remaining > 1) + goto other_parity_scan; + } return -EADDRNOTAVAIL; ok: @@ -1109,8 +1114,8 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row, * on low contention the randomness is maximal and on high contention * it may be inexistent. */ - i = max_t(int, i, get_random_u32_below(8) * 2); - WRITE_ONCE(table_perturb[index], READ_ONCE(table_perturb[index]) + i + 2); + i = max_t(int, i, get_random_u32_below(8) * step); + WRITE_ONCE(table_perturb[index], READ_ONCE(table_perturb[index]) + i + step); /* Head lock still held and bh's disabled */ inet_bind_hash(sk, tb, tb2, port); diff --git a/net/ipv4/ip_sockglue.c b/net/ipv4/ip_sockglue.c index d7d13940774e83..66247e8b429e43 100644 --- a/net/ipv4/ip_sockglue.c +++ b/net/ipv4/ip_sockglue.c @@ -775,7 +775,7 @@ static int ip_set_mcast_msfilter(struct sock *sk, sockptr_t optval, int optlen) if (optlen < GROUP_FILTER_SIZE(0)) return -EINVAL; - if (optlen > READ_ONCE(sysctl_optmem_max)) + if (optlen > READ_ONCE(sock_net(sk)->core.sysctl_optmem_max)) return -ENOBUFS; gsf = memdup_sockptr(optval, optlen); @@ -811,7 +811,7 @@ static int compat_ip_set_mcast_msfilter(struct sock *sk, sockptr_t optval, if (optlen < size0) return -EINVAL; - if (optlen > READ_ONCE(sysctl_optmem_max) - 4) + if (optlen > READ_ONCE(sock_net(sk)->core.sysctl_optmem_max) - 4) return -ENOBUFS; p = kmalloc(optlen + 4, GFP_KERNEL); @@ -1254,7 +1254,7 @@ int do_ip_setsockopt(struct sock *sk, int level, int optname, if (optlen < IP_MSFILTER_SIZE(0)) goto e_inval; - if (optlen > READ_ONCE(sysctl_optmem_max)) { + if (optlen > READ_ONCE(net->core.sysctl_optmem_max)) { err = -ENOBUFS; break; } diff --git a/net/ipv4/ipmr.c b/net/ipv4/ipmr.c index 9e222a57bc2b47..0063a237253bf4 100644 --- a/net/ipv4/ipmr.c +++ b/net/ipv4/ipmr.c @@ -1025,6 +1025,10 @@ static int ipmr_cache_report(const struct mr_table *mrt, struct sk_buff *skb; int ret; + mroute_sk = rcu_dereference(mrt->mroute_sk); + if (!mroute_sk) + return -EINVAL; + if (assert == IGMPMSG_WHOLEPKT || assert == IGMPMSG_WRVIFWHOLE) skb = skb_realloc_headroom(pkt, sizeof(struct iphdr)); else @@ -1069,7 +1073,8 @@ static int ipmr_cache_report(const struct mr_table *mrt, msg = (struct igmpmsg *)skb_network_header(skb); msg->im_vif = vifi; msg->im_vif_hi = vifi >> 8; - skb_dst_set(skb, dst_clone(skb_dst(pkt))); + ipv4_pktinfo_prepare(mroute_sk, pkt); + memcpy(skb->cb, pkt->cb, sizeof(skb->cb)); /* Add our header */ igmp = skb_put(skb, sizeof(struct igmphdr)); igmp->type = assert; @@ -1079,12 +1084,6 @@ static int ipmr_cache_report(const struct mr_table *mrt, skb->transport_header = skb->network_header; } - mroute_sk = rcu_dereference(mrt->mroute_sk); - if (!mroute_sk) { - kfree_skb(skb); - return -EINVAL; - } - igmpmsg_netlink_event(mrt, skb); /* Deliver to mrouted */ diff --git a/net/ipv6/ipv6_sockglue.c b/net/ipv6/ipv6_sockglue.c index 9e8ebda170f14f..56c3c467f9deb9 100644 --- a/net/ipv6/ipv6_sockglue.c +++ b/net/ipv6/ipv6_sockglue.c @@ -210,7 +210,7 @@ static int ipv6_set_mcast_msfilter(struct sock *sk, sockptr_t optval, if (optlen < GROUP_FILTER_SIZE(0)) return -EINVAL; - if (optlen > READ_ONCE(sysctl_optmem_max)) + if (optlen > READ_ONCE(sock_net(sk)->core.sysctl_optmem_max)) return -ENOBUFS; gsf = memdup_sockptr(optval, optlen); @@ -244,7 +244,7 @@ static int compat_ipv6_set_mcast_msfilter(struct sock *sk, sockptr_t optval, if (optlen < size0) return -EINVAL; - if (optlen > READ_ONCE(sysctl_optmem_max) - 4) + if (optlen > READ_ONCE(sock_net(sk)->core.sysctl_optmem_max) - 4) return -ENOBUFS; p = kmalloc(optlen + 4, GFP_KERNEL); diff --git a/net/mac80211/Makefile b/net/mac80211/Makefile index c9eb52768133d0..4406b4f8f3b974 100644 --- a/net/mac80211/Makefile +++ b/net/mac80211/Makefile @@ -67,4 +67,6 @@ mac80211-$(CONFIG_MAC80211_RC_MINSTREL) += $(rc80211_minstrel-y) obj-y += tests/ +mac80211-y += wbrf.o + ccflags-y += -DDEBUG diff --git a/net/mac80211/cfg.c b/net/mac80211/cfg.c index eb1d3ef843538c..489dd97f51724a 100644 --- a/net/mac80211/cfg.c +++ b/net/mac80211/cfg.c @@ -1270,7 +1270,7 @@ static int ieee80211_start_ap(struct wiphy *wiphy, struct net_device *dev, return -EALREADY; if (params->smps_mode != NL80211_SMPS_OFF) - return -ENOTSUPP; + return -EOPNOTSUPP; link->smps_mode = IEEE80211_SMPS_OFF; @@ -2556,7 +2556,7 @@ static int ieee80211_update_mesh_config(struct wiphy *wiphy, * devices that report signal in dBm. */ if (!ieee80211_hw_check(&sdata->local->hw, SIGNAL_DBM)) - return -ENOTSUPP; + return -EOPNOTSUPP; conf->rssi_threshold = nconf->rssi_threshold; } if (_chg_mesh_attr(NL80211_MESHCONF_HT_OPMODE, mask)) { diff --git a/net/mac80211/chan.c b/net/mac80211/chan.c index 1d928f29ad6f50..ef4c2cebc08089 100644 --- a/net/mac80211/chan.c +++ b/net/mac80211/chan.c @@ -507,11 +507,16 @@ static void _ieee80211_change_chanctx(struct ieee80211_local *local, WARN_ON(!cfg80211_chandef_compatible(&ctx->conf.def, chandef)); + ieee80211_remove_wbrf(local, &ctx->conf.def); + ctx->conf.def = *chandef; /* check if min chanctx also changed */ changed = IEEE80211_CHANCTX_CHANGE_WIDTH | _ieee80211_recalc_chanctx_min_def(local, ctx, rsvd_for); + + ieee80211_add_wbrf(local, &ctx->conf.def); + drv_change_chanctx(local, ctx, changed); if (!local->use_chanctx) { @@ -667,6 +672,8 @@ static int ieee80211_add_chanctx(struct ieee80211_local *local, lockdep_assert_wiphy(local->hw.wiphy); + ieee80211_add_wbrf(local, &ctx->conf.def); + if (!local->use_chanctx) local->hw.conf.radar_enabled = ctx->conf.radar_enabled; @@ -746,6 +753,8 @@ static void ieee80211_del_chanctx(struct ieee80211_local *local, } ieee80211_recalc_idle(local); + + ieee80211_remove_wbrf(local, &ctx->conf.def); } static void ieee80211_free_chanctx(struct ieee80211_local *local, @@ -858,7 +867,7 @@ static int ieee80211_assign_link_chanctx(struct ieee80211_link_data *link, int ret = 0; if (WARN_ON(sdata->vif.type == NL80211_IFTYPE_NAN)) - return -ENOTSUPP; + return -EOPNOTSUPP; conf = rcu_dereference_protected(link->conf->chanctx_conf, lockdep_is_held(&local->hw.wiphy->mtx)); @@ -1106,7 +1115,7 @@ int ieee80211_link_reserve_chanctx(struct ieee80211_link_data *link, curr_ctx = ieee80211_link_get_chanctx(link); if (curr_ctx && local->use_chanctx && !local->ops->switch_vif_chanctx) - return -ENOTSUPP; + return -EOPNOTSUPP; new_ctx = ieee80211_find_reservation_chanctx(local, chandef, mode); if (!new_ctx) { diff --git a/net/mac80211/debugfs.c b/net/mac80211/debugfs.c index b575ae90e57f86..74be49191e7043 100644 --- a/net/mac80211/debugfs.c +++ b/net/mac80211/debugfs.c @@ -497,6 +497,7 @@ static const char *hw_flag_names[] = { FLAG(SUPPORTS_CONC_MON_RX_DECAP), FLAG(DETECTS_COLOR_COLLISION), FLAG(MLO_MCAST_MULTI_LINK_TX), + FLAG(DISALLOW_PUNCTURING), #undef FLAG }; diff --git a/net/mac80211/driver-ops.h b/net/mac80211/driver-ops.h index f690c385a345a6..e9219f9278750d 100644 --- a/net/mac80211/driver-ops.h +++ b/net/mac80211/driver-ops.h @@ -805,7 +805,7 @@ drv_cancel_remain_on_channel(struct ieee80211_local *local, static inline int drv_set_ringparam(struct ieee80211_local *local, u32 tx, u32 rx) { - int ret = -ENOTSUPP; + int ret = -EOPNOTSUPP; might_sleep(); lockdep_assert_wiphy(local->hw.wiphy); diff --git a/net/mac80211/ibss.c b/net/mac80211/ibss.c index 8b1e02f2f9ae1c..8f2b445a5ec3be 100644 --- a/net/mac80211/ibss.c +++ b/net/mac80211/ibss.c @@ -485,7 +485,7 @@ int ieee80211_ibss_csa_beacon(struct ieee80211_sub_if_data *sdata, ifibss->ssid_len, IEEE80211_BSS_TYPE_IBSS, IEEE80211_PRIVACY(ifibss->privacy)); - if (WARN_ON(!cbss)) + if (unlikely(!cbss)) return -EINVAL; rcu_read_lock(); diff --git a/net/mac80211/ieee80211_i.h b/net/mac80211/ieee80211_i.h index 84df104f272b01..29312f6638a15a 100644 --- a/net/mac80211/ieee80211_i.h +++ b/net/mac80211/ieee80211_i.h @@ -92,11 +92,14 @@ enum ieee80211_status_data { IEEE80211_STATUS_SUBDATA_MASK = 0xff0, }; -/* - * Keep a station's queues on the active list for deficit accounting purposes - * if it was active or queued during the last 100ms - */ -#define AIRTIME_ACTIVE_DURATION (HZ / 10) +static inline bool +ieee80211_sta_keep_active(struct sta_info *sta, u8 ac) +{ + /* Keep a station's queues on the active list for deficit accounting + * purposes if it was active or queued during the last 100ms. + */ + return time_before_eq(jiffies, sta->airtime[ac].last_active + HZ / 10); +} struct ieee80211_bss { u32 device_ts_beacon, device_ts_presp; @@ -1559,6 +1562,8 @@ struct ieee80211_local { /* extended capabilities provided by mac80211 */ u8 ext_capa[8]; + + bool wbrf_supported; }; static inline struct ieee80211_sub_if_data * @@ -2600,4 +2605,9 @@ ieee80211_eht_cap_ie_to_sta_eht_cap(struct ieee80211_sub_if_data *sdata, const struct ieee80211_eht_cap_elem *eht_cap_ie_elem, u8 eht_cap_len, struct link_sta_info *link_sta); + +void ieee80211_check_wbrf_support(struct ieee80211_local *local); +void ieee80211_add_wbrf(struct ieee80211_local *local, struct cfg80211_chan_def *chandef); +void ieee80211_remove_wbrf(struct ieee80211_local *local, struct cfg80211_chan_def *chandef); + #endif /* IEEE80211_I_H */ diff --git a/net/mac80211/main.c b/net/mac80211/main.c index 033a5261ac3a3d..f2ece77935739f 100644 --- a/net/mac80211/main.c +++ b/net/mac80211/main.c @@ -1405,6 +1405,8 @@ int ieee80211_register_hw(struct ieee80211_hw *hw) debugfs_hw_add(local); rate_control_add_debugfs(local); + ieee80211_check_wbrf_support(local); + rtnl_lock(); wiphy_lock(hw->wiphy); diff --git a/net/mac80211/mesh_hwmp.c b/net/mac80211/mesh_hwmp.c index 775d52561c54a0..024f48db6b05b8 100644 --- a/net/mac80211/mesh_hwmp.c +++ b/net/mac80211/mesh_hwmp.c @@ -151,7 +151,7 @@ static int mesh_path_sel_frame_tx(enum mpath_frame_type action, u8 flags, break; default: kfree_skb(skb); - return -ENOTSUPP; + return -EOPNOTSUPP; } *pos++ = ie_len; *pos++ = flags; diff --git a/net/mac80211/mesh_pathtbl.c b/net/mac80211/mesh_pathtbl.c index 8a3f44ce3e0416..735edde1bd8199 100644 --- a/net/mac80211/mesh_pathtbl.c +++ b/net/mac80211/mesh_pathtbl.c @@ -676,10 +676,10 @@ struct mesh_path *mesh_path_add(struct ieee80211_sub_if_data *sdata, if (ether_addr_equal(dst, sdata->vif.addr)) /* never add ourselves as neighbours */ - return ERR_PTR(-ENOTSUPP); + return ERR_PTR(-EOPNOTSUPP); if (is_multicast_ether_addr(dst)) - return ERR_PTR(-ENOTSUPP); + return ERR_PTR(-EOPNOTSUPP); if (atomic_add_unless(&sdata->u.mesh.mpaths, 1, MESH_MAX_MPATHS) == 0) return ERR_PTR(-ENOSPC); @@ -719,10 +719,10 @@ int mpp_path_add(struct ieee80211_sub_if_data *sdata, if (ether_addr_equal(dst, sdata->vif.addr)) /* never add ourselves as neighbours */ - return -ENOTSUPP; + return -EOPNOTSUPP; if (is_multicast_ether_addr(dst)) - return -ENOTSUPP; + return -EOPNOTSUPP; new_mpath = mesh_path_new(sdata, dst, GFP_ATOMIC); diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c index c8998cf01b7a54..40a4fbfff5304d 100644 --- a/net/mac80211/mlme.c +++ b/net/mac80211/mlme.c @@ -135,6 +135,7 @@ ieee80211_handle_puncturing_bitmap(struct ieee80211_link_data *link, u16 bitmap, u64 *changed) { struct cfg80211_chan_def *chandef = &link->conf->chandef; + struct ieee80211_local *local = link->sdata->local; u16 extracted; u64 _changed = 0; @@ -147,7 +148,9 @@ ieee80211_handle_puncturing_bitmap(struct ieee80211_link_data *link, bitmap); if (cfg80211_valid_disable_subchannel_bitmap(&bitmap, - chandef)) + chandef) && + !(bitmap && ieee80211_hw_check(&local->hw, + DISALLOW_PUNCTURING))) break; link->u.mgd.conn_flags |= ieee80211_chandef_downgrade(chandef); @@ -1382,7 +1385,7 @@ static int ieee80211_send_assoc(struct ieee80211_sub_if_data *sdata) struct ieee80211_mgmt *mgmt; u8 *pos, qos_info, *ie_start; size_t offset, noffset; - u16 capab = WLAN_CAPABILITY_ESS, link_capab; + u16 capab = 0, link_capab; __le16 listen_int; struct element *ext_capa = NULL; enum nl80211_iftype iftype = ieee80211_vif_type_p2p(&sdata->vif); @@ -1529,6 +1532,17 @@ static int ieee80211_send_assoc(struct ieee80211_sub_if_data *sdata) *pos++ = assoc_data->ssid_len; memcpy(pos, assoc_data->ssid, assoc_data->ssid_len); + /* + * This bit is technically reserved, so it shouldn't matter for either + * the AP or us, but it also means we shouldn't set it. However, we've + * always set it in the past, and apparently some EHT APs check that + * we don't set it. To avoid interoperability issues with old APs that + * for some reason check it and want it to be set, set the bit for all + * pre-EHT connections as we used to do. + */ + if (link->u.mgd.conn_flags & IEEE80211_CONN_DISABLE_EHT) + capab |= WLAN_CAPABILITY_ESS; + /* add the elements for the assoc (main) link */ link_capab = capab; offset = ieee80211_assoc_link_elems(sdata, skb, &link_capab, @@ -5682,6 +5696,7 @@ static bool ieee80211_config_puncturing(struct ieee80211_link_data *link, const struct ieee80211_eht_operation *eht_oper, u64 *changed) { + struct ieee80211_local *local = link->sdata->local; u16 bitmap = 0, extracted; if ((eht_oper->params & IEEE80211_EHT_OPER_INFO_PRESENT) && @@ -5713,6 +5728,9 @@ static bool ieee80211_config_puncturing(struct ieee80211_link_data *link, return false; } + if (bitmap && ieee80211_hw_check(&local->hw, DISALLOW_PUNCTURING)) + return false; + ieee80211_handle_puncturing_bitmap(link, eht_oper, bitmap, changed); return true; } @@ -7586,7 +7604,8 @@ ieee80211_setup_assoc_link(struct ieee80211_sub_if_data *sdata, bitmap = get_unaligned_le16(disable_subchannel_bitmap); if (cfg80211_valid_disable_subchannel_bitmap(&bitmap, - &link->conf->chandef)) + &link->conf->chandef) && + !(bitmap && ieee80211_hw_check(&local->hw, DISALLOW_PUNCTURING))) ieee80211_handle_puncturing_bitmap(link, eht_oper, bitmap, diff --git a/net/mac80211/rx.c b/net/mac80211/rx.c index 64352e4e6d0088..bbfdcb0ade72c0 100644 --- a/net/mac80211/rx.c +++ b/net/mac80211/rx.c @@ -920,7 +920,7 @@ static void ieee80211_parse_qos(struct ieee80211_rx_data *rx) * Drivers always need to pass packets that are aligned to two-byte boundaries * to the stack. * - * Additionally, should, if possible, align the payload data in a way that + * Additionally, they should, if possible, align the payload data in a way that * guarantees that the contained IP header is aligned to a four-byte * boundary. In the case of regular frames, this simply means aligning the * payload to a four-byte boundary (because either the IP header is directly @@ -936,7 +936,7 @@ static void ieee80211_parse_qos(struct ieee80211_rx_data *rx) * subframe to a length that is a multiple of four. * * Padding like Atheros hardware adds which is between the 802.11 header and - * the payload is not supported, the driver is required to move the 802.11 + * the payload is not supported; the driver is required to move the 802.11 * header to be directly in front of the payload in that case. */ static void ieee80211_verify_alignment(struct ieee80211_rx_data *rx) diff --git a/net/mac80211/scan.c b/net/mac80211/scan.c index 1d98877647d85c..645355e5f1bc7b 100644 --- a/net/mac80211/scan.c +++ b/net/mac80211/scan.c @@ -1289,7 +1289,7 @@ int __ieee80211_request_sched_scan_start(struct ieee80211_sub_if_data *sdata, iebufsz = local->scan_ies_len + req->ie_len; if (!local->ops->sched_scan_start) - return -ENOTSUPP; + return -EOPNOTSUPP; for (i = 0; i < NUM_NL80211_BANDS; i++) { if (local->hw.wiphy->bands[i]) { @@ -1354,7 +1354,7 @@ int ieee80211_request_sched_scan_stop(struct ieee80211_local *local) lockdep_assert_wiphy(local->hw.wiphy); if (!local->ops->sched_scan_stop) - return -ENOTSUPP; + return -EOPNOTSUPP; /* We don't want to restart sched scan anymore. */ RCU_INIT_POINTER(local->sched_scan_req, NULL); diff --git a/net/mac80211/sta_info.c b/net/mac80211/sta_info.c index 0ba613dd1cc478..bf1adcd96b4113 100644 --- a/net/mac80211/sta_info.c +++ b/net/mac80211/sta_info.c @@ -40,7 +40,7 @@ * either sta_info_insert() or sta_info_insert_rcu(); only in the latter * case (which acquires an rcu read section but must not be called from * within one) will the pointer still be valid after the call. Note that - * the caller may not do much with the STA info before inserting it, in + * the caller may not do much with the STA info before inserting it; in * particular, it may not start any mesh peer link management or add * encryption keys. * @@ -58,7 +58,7 @@ * In order to remove a STA info structure, various sta_info_destroy_*() * calls are available. * - * There is no concept of ownership on a STA entry, each structure is + * There is no concept of ownership on a STA entry; each structure is * owned by the global hash table/list until it is removed. All users of * the structure need to be RCU protected so that the structure won't be * freed before they are done using it. @@ -2268,7 +2268,6 @@ void ieee80211_sta_register_airtime(struct ieee80211_sta *pubsta, u8 tid, struct ieee80211_local *local = sta->sdata->local; u8 ac = ieee80211_ac_from_tid(tid); u32 airtime = 0; - u32 diff; if (sta->local->airtime_flags & AIRTIME_USE_TX) airtime += tx_airtime; @@ -2279,8 +2278,7 @@ void ieee80211_sta_register_airtime(struct ieee80211_sta *pubsta, u8 tid, sta->airtime[ac].tx_airtime += tx_airtime; sta->airtime[ac].rx_airtime += rx_airtime; - diff = (u32)jiffies - sta->airtime[ac].last_active; - if (diff <= AIRTIME_ACTIVE_DURATION) + if (ieee80211_sta_keep_active(sta, ac)) sta->airtime[ac].deficit -= airtime; spin_unlock_bh(&local->active_txq_lock[ac]); diff --git a/net/mac80211/sta_info.h b/net/mac80211/sta_info.h index 7acf2223e47aa2..5ef1554f991f6d 100644 --- a/net/mac80211/sta_info.h +++ b/net/mac80211/sta_info.h @@ -138,7 +138,7 @@ enum ieee80211_agg_stop_reason { struct airtime_info { u64 rx_airtime; u64 tx_airtime; - u32 last_active; + unsigned long last_active; s32 deficit; atomic_t aql_tx_pending; /* Estimated airtime for frames pending */ u32 aql_limit_low; diff --git a/net/mac80211/tdls.c b/net/mac80211/tdls.c index 05a7dff69fe94f..49730b424141a8 100644 --- a/net/mac80211/tdls.c +++ b/net/mac80211/tdls.c @@ -1001,7 +1001,7 @@ ieee80211_tdls_build_mgmt_packet_data(struct ieee80211_sub_if_data *sdata, skb); break; default: - ret = -ENOTSUPP; + ret = -EOPNOTSUPP; break; } @@ -1071,7 +1071,7 @@ ieee80211_tdls_prep_mgmt_packet(struct wiphy *wiphy, struct net_device *dev, /* any value is ok */ break; default: - ret = -ENOTSUPP; + ret = -EOPNOTSUPP; break; } @@ -1177,7 +1177,7 @@ ieee80211_tdls_mgmt_setup(struct wiphy *wiphy, struct net_device *dev, smps_mode != IEEE80211_SMPS_OFF) { tdls_dbg(sdata, "Aborting TDLS setup due to SMPS mode %d\n", smps_mode); - return -ENOTSUPP; + return -EOPNOTSUPP; } lockdep_assert_wiphy(local->hw.wiphy); @@ -1289,7 +1289,7 @@ int ieee80211_tdls_mgmt(struct wiphy *wiphy, struct net_device *dev, int ret; if (!(wiphy->flags & WIPHY_FLAG_SUPPORTS_TDLS)) - return -ENOTSUPP; + return -EOPNOTSUPP; /* make sure we are in managed mode, and associated */ if (sdata->vif.type != NL80211_IFTYPE_STATION || @@ -1446,7 +1446,7 @@ int ieee80211_tdls_oper(struct wiphy *wiphy, struct net_device *dev, lockdep_assert_wiphy(local->hw.wiphy); if (!(wiphy->flags & WIPHY_FLAG_SUPPORTS_TDLS)) - return -ENOTSUPP; + return -EOPNOTSUPP; if (sdata->vif.type != NL80211_IFTYPE_STATION) return -EINVAL; @@ -1459,7 +1459,7 @@ int ieee80211_tdls_oper(struct wiphy *wiphy, struct net_device *dev, case NL80211_TDLS_SETUP: case NL80211_TDLS_DISCOVERY_REQ: /* We don't support in-driver setup/teardown/discovery */ - return -ENOTSUPP; + return -EOPNOTSUPP; } /* protect possible bss_conf changes and avoid concurrency in @@ -1510,7 +1510,7 @@ int ieee80211_tdls_oper(struct wiphy *wiphy, struct net_device *dev, return ret; break; default: - return -ENOTSUPP; + return -EOPNOTSUPP; } if (ether_addr_equal(sdata->u.mgd.tdls_peer, peer)) { @@ -1673,7 +1673,7 @@ ieee80211_tdls_channel_switch(struct wiphy *wiphy, struct net_device *dev, if (!test_sta_flag(sta, WLAN_STA_TDLS_CHAN_SWITCH)) { tdls_dbg(sdata, "TDLS channel switch unsupported by %pM\n", addr); - ret = -ENOTSUPP; + ret = -EOPNOTSUPP; goto out; } @@ -1993,7 +1993,7 @@ ieee80211_process_tdls_channel_switch_req(struct ieee80211_sub_if_data *sdata, if (!sta->sta.deflink.ht_cap.ht_supported && elems->sec_chan_offs && elems->sec_chan_offs->sec_chan_offs) { tdls_dbg(sdata, "TDLS chan switch - wide chan unsupported\n"); - ret = -ENOTSUPP; + ret = -EOPNOTSUPP; goto out; } diff --git a/net/mac80211/tx.c b/net/mac80211/tx.c index ed4fdf655343f2..314998fdb1a5a4 100644 --- a/net/mac80211/tx.c +++ b/net/mac80211/tx.c @@ -4013,14 +4013,13 @@ ieee80211_txq_set_active(struct txq_info *txqi) return; sta = container_of(txqi->txq.sta, struct sta_info, sta); - sta->airtime[txqi->txq.ac].last_active = (u32)jiffies; + sta->airtime[txqi->txq.ac].last_active = jiffies; } static bool ieee80211_txq_keep_active(struct txq_info *txqi) { struct sta_info *sta; - u32 diff; if (!txqi->txq.sta) return false; @@ -4029,9 +4028,7 @@ ieee80211_txq_keep_active(struct txq_info *txqi) if (ieee80211_sta_deficit(sta, txqi->txq.ac) >= 0) return false; - diff = (u32)jiffies - sta->airtime[txqi->txq.ac].last_active; - - return diff <= AIRTIME_ACTIVE_DURATION; + return ieee80211_sta_keep_active(sta, txqi->txq.ac); } struct ieee80211_txq *ieee80211_next_txq(struct ieee80211_hw *hw, u8 ac) diff --git a/net/mac80211/wbrf.c b/net/mac80211/wbrf.c new file mode 100644 index 00000000000000..a05c5b971789c7 --- /dev/null +++ b/net/mac80211/wbrf.c @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Wifi Band Exclusion Interface for WLAN + * Copyright (C) 2023 Advanced Micro Devices + * + */ + +#include +#include +#include +#include "ieee80211_i.h" + +void ieee80211_check_wbrf_support(struct ieee80211_local *local) +{ + struct wiphy *wiphy = local->hw.wiphy; + struct device *dev; + + if (!wiphy) + return; + + dev = wiphy->dev.parent; + if (!dev) + return; + + local->wbrf_supported = acpi_amd_wbrf_supported_producer(dev); + dev_dbg(dev, "WBRF is %s supported\n", + local->wbrf_supported ? "" : "not"); +} + +static void get_chan_freq_boundary(u32 center_freq, u32 bandwidth, u64 *start, u64 *end) +{ + bandwidth *= KHZ_PER_MHZ; + center_freq *= KHZ_PER_MHZ; + + *start = center_freq - bandwidth / 2; + *end = center_freq + bandwidth / 2; + + /* Frequency in Hz is expected */ + *start = *start * HZ_PER_KHZ; + *end = *end * HZ_PER_KHZ; +} + +static void get_ranges_from_chandef(struct cfg80211_chan_def *chandef, + struct wbrf_ranges_in_out *ranges_in) +{ + u64 start_freq1, end_freq1; + u64 start_freq2, end_freq2; + int bandwidth; + + bandwidth = nl80211_chan_width_to_mhz(chandef->width); + + get_chan_freq_boundary(chandef->center_freq1, bandwidth, &start_freq1, &end_freq1); + + ranges_in->band_list[0].start = start_freq1; + ranges_in->band_list[0].end = end_freq1; + ranges_in->num_of_ranges = 1; + + if (chandef->width == NL80211_CHAN_WIDTH_80P80) { + get_chan_freq_boundary(chandef->center_freq2, bandwidth, &start_freq2, &end_freq2); + + ranges_in->band_list[1].start = start_freq2; + ranges_in->band_list[1].end = end_freq2; + ranges_in->num_of_ranges++; + } +} + +void ieee80211_add_wbrf(struct ieee80211_local *local, struct cfg80211_chan_def *chandef) +{ + struct wbrf_ranges_in_out ranges_in = {0}; + struct device *dev; + + if (!local->wbrf_supported) + return; + + dev = local->hw.wiphy->dev.parent; + + get_ranges_from_chandef(chandef, &ranges_in); + + acpi_amd_wbrf_add_remove(dev, WBRF_RECORD_ADD, &ranges_in); +} + +void ieee80211_remove_wbrf(struct ieee80211_local *local, struct cfg80211_chan_def *chandef) +{ + struct wbrf_ranges_in_out ranges_in = {0}; + struct device *dev; + + if (!local->wbrf_supported) + return; + + dev = local->hw.wiphy->dev.parent; + + get_ranges_from_chandef(chandef, &ranges_in); + + acpi_amd_wbrf_add_remove(dev, WBRF_RECORD_REMOVE, &ranges_in); +} diff --git a/net/mptcp/mptcp_pm_gen.c b/net/mptcp/mptcp_pm_gen.c index a2325e70ddabd1..670da7822e6c91 100644 --- a/net/mptcp/mptcp_pm_gen.c +++ b/net/mptcp/mptcp_pm_gen.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) /* Do not edit directly, auto-generated from: */ -/* Documentation/netlink/specs/mptcp.yaml */ +/* Documentation/netlink/specs/mptcp_pm.yaml */ /* YNL-GEN kernel source */ #include diff --git a/net/mptcp/mptcp_pm_gen.h b/net/mptcp/mptcp_pm_gen.h index 10579d18458708..ac9fc7225b6a09 100644 --- a/net/mptcp/mptcp_pm_gen.h +++ b/net/mptcp/mptcp_pm_gen.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */ /* Do not edit directly, auto-generated from: */ -/* Documentation/netlink/specs/mptcp.yaml */ +/* Documentation/netlink/specs/mptcp_pm.yaml */ /* YNL-GEN kernel header */ #ifndef _LINUX_MPTCP_PM_GEN_H diff --git a/net/netfilter/nf_bpf_link.c b/net/netfilter/nf_bpf_link.c index 0e4beae421f830..5257d5e7eb09d8 100644 --- a/net/netfilter/nf_bpf_link.c +++ b/net/netfilter/nf_bpf_link.c @@ -314,7 +314,7 @@ static bool nf_is_valid_access(int off, int size, enum bpf_access_type type, static const struct bpf_func_proto * bpf_nf_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { - return bpf_base_func_proto(func_id); + return bpf_base_func_proto(func_id, prog); } const struct bpf_verifier_ops netfilter_verifier_ops = { diff --git a/net/vmw_vsock/af_vsock.c b/net/vmw_vsock/af_vsock.c index 816725af281f38..54ba7316f8085d 100644 --- a/net/vmw_vsock/af_vsock.c +++ b/net/vmw_vsock/af_vsock.c @@ -2264,8 +2264,13 @@ static int vsock_set_rcvlowat(struct sock *sk, int val) transport = vsk->transport; - if (transport && transport->set_rcvlowat) - return transport->set_rcvlowat(vsk, val); + if (transport && transport->notify_set_rcvlowat) { + int err; + + err = transport->notify_set_rcvlowat(vsk, val); + if (err) + return err; + } WRITE_ONCE(sk->sk_rcvlowat, val ? : 1); return 0; diff --git a/net/vmw_vsock/hyperv_transport.c b/net/vmw_vsock/hyperv_transport.c index 7cb1a9d2cdb4f8..e2157e38721770 100644 --- a/net/vmw_vsock/hyperv_transport.c +++ b/net/vmw_vsock/hyperv_transport.c @@ -816,7 +816,7 @@ int hvs_notify_send_post_enqueue(struct vsock_sock *vsk, ssize_t written, } static -int hvs_set_rcvlowat(struct vsock_sock *vsk, int val) +int hvs_notify_set_rcvlowat(struct vsock_sock *vsk, int val) { return -EOPNOTSUPP; } @@ -856,7 +856,7 @@ static struct vsock_transport hvs_transport = { .notify_send_pre_enqueue = hvs_notify_send_pre_enqueue, .notify_send_post_enqueue = hvs_notify_send_post_enqueue, - .set_rcvlowat = hvs_set_rcvlowat + .notify_set_rcvlowat = hvs_notify_set_rcvlowat }; static bool hvs_check_transport(struct vsock_sock *vsk) diff --git a/net/vmw_vsock/virtio_transport.c b/net/vmw_vsock/virtio_transport.c index af5bab1acee17d..f495b9e5186b2e 100644 --- a/net/vmw_vsock/virtio_transport.c +++ b/net/vmw_vsock/virtio_transport.c @@ -537,6 +537,7 @@ static struct virtio_transport virtio_transport = { .notify_send_pre_enqueue = virtio_transport_notify_send_pre_enqueue, .notify_send_post_enqueue = virtio_transport_notify_send_post_enqueue, .notify_buffer_size = virtio_transport_notify_buffer_size, + .notify_set_rcvlowat = virtio_transport_notify_set_rcvlowat, .read_skb = virtio_transport_read_skb, }, diff --git a/net/vmw_vsock/virtio_transport_common.c b/net/vmw_vsock/virtio_transport_common.c index 6df246b5326063..16ff976a86e3ed 100644 --- a/net/vmw_vsock/virtio_transport_common.c +++ b/net/vmw_vsock/virtio_transport_common.c @@ -557,6 +557,8 @@ virtio_transport_stream_do_dequeue(struct vsock_sock *vsk, struct virtio_vsock_sock *vvs = vsk->trans; size_t bytes, total = 0; struct sk_buff *skb; + u32 fwd_cnt_delta; + bool low_rx_bytes; int err = -EFAULT; u32 free_space; @@ -600,7 +602,10 @@ virtio_transport_stream_do_dequeue(struct vsock_sock *vsk, } } - free_space = vvs->buf_alloc - (vvs->fwd_cnt - vvs->last_fwd_cnt); + fwd_cnt_delta = vvs->fwd_cnt - vvs->last_fwd_cnt; + free_space = vvs->buf_alloc - fwd_cnt_delta; + low_rx_bytes = (vvs->rx_bytes < + sock_rcvlowat(sk_vsock(vsk), 0, INT_MAX)); spin_unlock_bh(&vvs->rx_lock); @@ -610,9 +615,11 @@ virtio_transport_stream_do_dequeue(struct vsock_sock *vsk, * too high causes extra messages. Too low causes transmitter * stalls. As stalls are in theory more expensive than extra * messages, we set the limit to a high value. TODO: experiment - * with different values. + * with different values. Also send credit update message when + * number of bytes in rx queue is not enough to wake up reader. */ - if (free_space < VIRTIO_VSOCK_MAX_PKT_BUF_SIZE) + if (fwd_cnt_delta && + (free_space < VIRTIO_VSOCK_MAX_PKT_BUF_SIZE || low_rx_bytes)) virtio_transport_send_credit_update(vsk); return total; @@ -1683,6 +1690,36 @@ int virtio_transport_read_skb(struct vsock_sock *vsk, skb_read_actor_t recv_acto } EXPORT_SYMBOL_GPL(virtio_transport_read_skb); +int virtio_transport_notify_set_rcvlowat(struct vsock_sock *vsk, int val) +{ + struct virtio_vsock_sock *vvs = vsk->trans; + bool send_update; + + spin_lock_bh(&vvs->rx_lock); + + /* If number of available bytes is less than new SO_RCVLOWAT value, + * kick sender to send more data, because sender may sleep in its + * 'send()' syscall waiting for enough space at our side. Also + * don't send credit update when peer already knows actual value - + * such transmission will be useless. + */ + send_update = (vvs->rx_bytes < val) && + (vvs->fwd_cnt != vvs->last_fwd_cnt); + + spin_unlock_bh(&vvs->rx_lock); + + if (send_update) { + int err; + + err = virtio_transport_send_credit_update(vsk); + if (err < 0) + return err; + } + + return 0; +} +EXPORT_SYMBOL_GPL(virtio_transport_notify_set_rcvlowat); + MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Asias He"); MODULE_DESCRIPTION("common code for virtio vsock"); diff --git a/net/vmw_vsock/vsock_loopback.c b/net/vmw_vsock/vsock_loopback.c index 0486401674117d..6dea6119f5b281 100644 --- a/net/vmw_vsock/vsock_loopback.c +++ b/net/vmw_vsock/vsock_loopback.c @@ -96,6 +96,7 @@ static struct virtio_transport loopback_transport = { .notify_send_pre_enqueue = virtio_transport_notify_send_pre_enqueue, .notify_send_post_enqueue = virtio_transport_notify_send_post_enqueue, .notify_buffer_size = virtio_transport_notify_buffer_size, + .notify_set_rcvlowat = virtio_transport_notify_set_rcvlowat, .read_skb = virtio_transport_read_skb, }, diff --git a/net/wireless/Makefile b/net/wireless/Makefile index 089c841528c854..72074fd36df4c8 100644 --- a/net/wireless/Makefile +++ b/net/wireless/Makefile @@ -25,7 +25,7 @@ ifneq ($(CONFIG_CFG80211_EXTRA_REGDB_KEYDIR),) cfg80211-y += extra-certs.o endif -$(obj)/shipped-certs.c: $(wildcard $(srctree)/$(src)/certs/*.hex) +$(obj)/shipped-certs.c: $(sort $(wildcard $(srctree)/$(src)/certs/*.hex)) @$(kecho) " GEN $@" $(Q)(echo '#include "reg.h"'; \ echo 'const u8 shipped_regdb_certs[] = {'; \ @@ -35,7 +35,7 @@ $(obj)/shipped-certs.c: $(wildcard $(srctree)/$(src)/certs/*.hex) ) > $@ $(obj)/extra-certs.c: $(CONFIG_CFG80211_EXTRA_REGDB_KEYDIR) \ - $(wildcard $(CONFIG_CFG80211_EXTRA_REGDB_KEYDIR)/*.x509) + $(sort $(wildcard $(CONFIG_CFG80211_EXTRA_REGDB_KEYDIR)/*.x509)) @$(kecho) " GEN $@" $(Q)(set -e; \ allf=""; \ diff --git a/net/wireless/chan.c b/net/wireless/chan.c index 2d21e423abdb0b..dfb4893421d77b 100644 --- a/net/wireless/chan.c +++ b/net/wireless/chan.c @@ -141,7 +141,7 @@ static bool cfg80211_edmg_chandef_valid(const struct cfg80211_chan_def *chandef) return true; } -static int nl80211_chan_width_to_mhz(enum nl80211_chan_width chan_width) +int nl80211_chan_width_to_mhz(enum nl80211_chan_width chan_width) { int mhz; @@ -190,6 +190,7 @@ static int nl80211_chan_width_to_mhz(enum nl80211_chan_width chan_width) } return mhz; } +EXPORT_SYMBOL(nl80211_chan_width_to_mhz); static int cfg80211_chandef_get_width(const struct cfg80211_chan_def *c) { diff --git a/net/wireless/core.h b/net/wireless/core.h index cb61d33d4f1ebc..1963958263d263 100644 --- a/net/wireless/core.h +++ b/net/wireless/core.h @@ -458,6 +458,9 @@ int cfg80211_scan(struct cfg80211_registered_device *rdev); extern struct work_struct cfg80211_disconnect_work; +#define NL80211_BSS_USE_FOR_ALL (NL80211_BSS_USE_FOR_NORMAL | \ + NL80211_BSS_USE_FOR_MLD_LINK) + void cfg80211_set_dfs_state(struct wiphy *wiphy, const struct cfg80211_chan_def *chandef, enum nl80211_dfs_state dfs_state); diff --git a/net/wireless/mlme.c b/net/wireless/mlme.c index bad9e4fd842fec..f635a8b6ca2ec0 100644 --- a/net/wireless/mlme.c +++ b/net/wireless/mlme.c @@ -22,7 +22,7 @@ void cfg80211_rx_assoc_resp(struct net_device *dev, - struct cfg80211_rx_assoc_resp_data *data) + const struct cfg80211_rx_assoc_resp_data *data) { struct wireless_dev *wdev = dev->ieee80211_ptr; struct wiphy *wiphy = wdev->wiphy; diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c index d0f499227c29f5..8b45fb420f4c0a 100644 --- a/net/wireless/nl80211.c +++ b/net/wireless/nl80211.c @@ -818,6 +818,9 @@ static const struct nla_policy nl80211_policy[NUM_NL80211_ATTR] = { [NL80211_ATTR_HW_TIMESTAMP_ENABLED] = { .type = NLA_FLAG }, [NL80211_ATTR_EMA_RNR_ELEMS] = { .type = NLA_NESTED }, [NL80211_ATTR_MLO_LINK_DISABLED] = { .type = NLA_FLAG }, + [NL80211_ATTR_BSS_DUMP_INCLUDE_USE_DATA] = { .type = NLA_FLAG }, + [NL80211_ATTR_MLO_TTLM_DLINK] = NLA_POLICY_EXACT_LEN(sizeof(u16) * 8), + [NL80211_ATTR_MLO_TTLM_ULINK] = NLA_POLICY_EXACT_LEN(sizeof(u16) * 8), }; /* policy for the key attributes */ @@ -4855,7 +4858,7 @@ static struct cfg80211_acl_data *parse_acl_data(struct wiphy *wiphy, return ERR_PTR(n_entries); if (n_entries > wiphy->max_acl_mac_addrs) - return ERR_PTR(-ENOTSUPP); + return ERR_PTR(-EOPNOTSUPP); acl = kzalloc(struct_size(acl, mac_addrs, n_entries), GFP_KERNEL); if (!acl) @@ -10410,6 +10413,15 @@ static int nl80211_send_bss(struct sk_buff *msg, struct netlink_callback *cb, break; } + if (nla_put_u32(msg, NL80211_BSS_USE_FOR, res->use_for)) + goto nla_put_failure; + + if (res->cannot_use_reasons && + nla_put_u64_64bit(msg, NL80211_BSS_CANNOT_USE_REASONS, + res->cannot_use_reasons, + NL80211_BSS_PAD)) + goto nla_put_failure; + nla_nest_end(msg, bss); genlmsg_end(msg, hdr); @@ -10427,15 +10439,27 @@ static int nl80211_dump_scan(struct sk_buff *skb, struct netlink_callback *cb) struct cfg80211_registered_device *rdev; struct cfg80211_internal_bss *scan; struct wireless_dev *wdev; + struct nlattr **attrbuf; int start = cb->args[2], idx = 0; + bool dump_include_use_data; int err; - err = nl80211_prepare_wdev_dump(cb, &rdev, &wdev, NULL); - if (err) + attrbuf = kcalloc(NUM_NL80211_ATTR, sizeof(*attrbuf), GFP_KERNEL); + if (!attrbuf) + return -ENOMEM; + + err = nl80211_prepare_wdev_dump(cb, &rdev, &wdev, attrbuf); + if (err) { + kfree(attrbuf); return err; + } /* nl80211_prepare_wdev_dump acquired it in the successful case */ __acquire(&rdev->wiphy.mtx); + dump_include_use_data = + attrbuf[NL80211_ATTR_BSS_DUMP_INCLUDE_USE_DATA]; + kfree(attrbuf); + spin_lock_bh(&rdev->bss_lock); /* @@ -10452,6 +10476,9 @@ static int nl80211_dump_scan(struct sk_buff *skb, struct netlink_callback *cb) list_for_each_entry(scan, &rdev->bss_list, list) { if (++idx <= start) continue; + if (!dump_include_use_data && + !(scan->pub.use_for & NL80211_BSS_USE_FOR_NORMAL)) + continue; if (nl80211_send_bss(skb, cb, cb->nlh->nlmsg_seq, NLM_F_MULTI, rdev, wdev, scan) < 0) { @@ -10903,12 +10930,13 @@ static int nl80211_crypto_settings(struct cfg80211_registered_device *rdev, static struct cfg80211_bss *nl80211_assoc_bss(struct cfg80211_registered_device *rdev, const u8 *ssid, int ssid_len, - struct nlattr **attrs) + struct nlattr **attrs, + int assoc_link_id, int link_id) { struct ieee80211_channel *chan; struct cfg80211_bss *bss; const u8 *bssid; - u32 freq; + u32 freq, use_for = 0; if (!attrs[NL80211_ATTR_MAC] || !attrs[NL80211_ATTR_WIPHY_FREQ]) return ERR_PTR(-EINVAL); @@ -10923,10 +10951,16 @@ static struct cfg80211_bss *nl80211_assoc_bss(struct cfg80211_registered_device if (!chan) return ERR_PTR(-EINVAL); - bss = cfg80211_get_bss(&rdev->wiphy, chan, bssid, - ssid, ssid_len, - IEEE80211_BSS_TYPE_ESS, - IEEE80211_PRIVACY_ANY); + if (assoc_link_id >= 0) + use_for = NL80211_BSS_USE_FOR_MLD_LINK; + if (assoc_link_id == link_id) + use_for |= NL80211_BSS_USE_FOR_NORMAL; + + bss = __cfg80211_get_bss(&rdev->wiphy, chan, bssid, + ssid, ssid_len, + IEEE80211_BSS_TYPE_ESS, + IEEE80211_PRIVACY_ANY, + use_for); if (!bss) return ERR_PTR(-ENOENT); @@ -11105,7 +11139,8 @@ static int nl80211_associate(struct sk_buff *skb, struct genl_info *info) goto free; } req.links[link_id].bss = - nl80211_assoc_bss(rdev, ssid, ssid_len, attrs); + nl80211_assoc_bss(rdev, ssid, ssid_len, attrs, + req.link_id, link_id); if (IS_ERR(req.links[link_id].bss)) { err = PTR_ERR(req.links[link_id].bss); req.links[link_id].bss = NULL; @@ -11170,7 +11205,8 @@ static int nl80211_associate(struct sk_buff *skb, struct genl_info *info) if (req.link_id >= 0) return -EINVAL; - req.bss = nl80211_assoc_bss(rdev, ssid, ssid_len, info->attrs); + req.bss = nl80211_assoc_bss(rdev, ssid, ssid_len, info->attrs, + -1, -1); if (IS_ERR(req.bss)) return PTR_ERR(req.bss); ap_addr = req.bss->bssid; @@ -12179,16 +12215,18 @@ static int nl80211_wiphy_netns(struct sk_buff *skb, struct genl_info *info) return err; } -static int nl80211_setdel_pmksa(struct sk_buff *skb, struct genl_info *info) +static int nl80211_set_pmksa(struct sk_buff *skb, struct genl_info *info) { struct cfg80211_registered_device *rdev = info->user_ptr[0]; - int (*rdev_ops)(struct wiphy *wiphy, struct net_device *dev, - struct cfg80211_pmksa *pmksa) = NULL; struct net_device *dev = info->user_ptr[1]; struct cfg80211_pmksa pmksa; + bool ap_pmksa_caching_support = false; memset(&pmksa, 0, sizeof(struct cfg80211_pmksa)); + ap_pmksa_caching_support = wiphy_ext_feature_isset(&rdev->wiphy, + NL80211_EXT_FEATURE_AP_PMKSA_CACHING); + if (!info->attrs[NL80211_ATTR_PMKID]) return -EINVAL; @@ -12197,16 +12235,15 @@ static int nl80211_setdel_pmksa(struct sk_buff *skb, struct genl_info *info) if (info->attrs[NL80211_ATTR_MAC]) { pmksa.bssid = nla_data(info->attrs[NL80211_ATTR_MAC]); } else if (info->attrs[NL80211_ATTR_SSID] && - info->attrs[NL80211_ATTR_FILS_CACHE_ID] && - (info->genlhdr->cmd == NL80211_CMD_DEL_PMKSA || - info->attrs[NL80211_ATTR_PMK])) { + info->attrs[NL80211_ATTR_FILS_CACHE_ID] && + info->attrs[NL80211_ATTR_PMK]) { pmksa.ssid = nla_data(info->attrs[NL80211_ATTR_SSID]); pmksa.ssid_len = nla_len(info->attrs[NL80211_ATTR_SSID]); - pmksa.cache_id = - nla_data(info->attrs[NL80211_ATTR_FILS_CACHE_ID]); + pmksa.cache_id = nla_data(info->attrs[NL80211_ATTR_FILS_CACHE_ID]); } else { return -EINVAL; } + if (info->attrs[NL80211_ATTR_PMK]) { pmksa.pmk = nla_data(info->attrs[NL80211_ATTR_PMK]); pmksa.pmk_len = nla_len(info->attrs[NL80211_ATTR_PMK]); @@ -12218,32 +12255,71 @@ static int nl80211_setdel_pmksa(struct sk_buff *skb, struct genl_info *info) if (info->attrs[NL80211_ATTR_PMK_REAUTH_THRESHOLD]) pmksa.pmk_reauth_threshold = - nla_get_u8( - info->attrs[NL80211_ATTR_PMK_REAUTH_THRESHOLD]); + nla_get_u8(info->attrs[NL80211_ATTR_PMK_REAUTH_THRESHOLD]); if (dev->ieee80211_ptr->iftype != NL80211_IFTYPE_STATION && dev->ieee80211_ptr->iftype != NL80211_IFTYPE_P2P_CLIENT && - !(dev->ieee80211_ptr->iftype == NL80211_IFTYPE_AP && - wiphy_ext_feature_isset(&rdev->wiphy, - NL80211_EXT_FEATURE_AP_PMKSA_CACHING))) + !((dev->ieee80211_ptr->iftype == NL80211_IFTYPE_AP || + dev->ieee80211_ptr->iftype == NL80211_IFTYPE_P2P_GO) && + ap_pmksa_caching_support)) return -EOPNOTSUPP; - switch (info->genlhdr->cmd) { - case NL80211_CMD_SET_PMKSA: - rdev_ops = rdev->ops->set_pmksa; - break; - case NL80211_CMD_DEL_PMKSA: - rdev_ops = rdev->ops->del_pmksa; - break; - default: - WARN_ON(1); - break; + if (!rdev->ops->set_pmksa) + return -EOPNOTSUPP; + + return rdev_set_pmksa(rdev, dev, &pmksa); +} + +static int nl80211_del_pmksa(struct sk_buff *skb, struct genl_info *info) +{ + struct cfg80211_registered_device *rdev = info->user_ptr[0]; + struct net_device *dev = info->user_ptr[1]; + struct cfg80211_pmksa pmksa; + bool sae_offload_support = false; + bool owe_offload_support = false; + bool ap_pmksa_caching_support = false; + + memset(&pmksa, 0, sizeof(struct cfg80211_pmksa)); + + sae_offload_support = wiphy_ext_feature_isset(&rdev->wiphy, + NL80211_EXT_FEATURE_SAE_OFFLOAD); + owe_offload_support = wiphy_ext_feature_isset(&rdev->wiphy, + NL80211_EXT_FEATURE_OWE_OFFLOAD); + ap_pmksa_caching_support = wiphy_ext_feature_isset(&rdev->wiphy, + NL80211_EXT_FEATURE_AP_PMKSA_CACHING); + + if (info->attrs[NL80211_ATTR_PMKID]) + pmksa.pmkid = nla_data(info->attrs[NL80211_ATTR_PMKID]); + + if (info->attrs[NL80211_ATTR_MAC]) { + pmksa.bssid = nla_data(info->attrs[NL80211_ATTR_MAC]); + } else if (info->attrs[NL80211_ATTR_SSID]) { + /* SSID based pmksa flush suppported only for FILS, + * OWE/SAE OFFLOAD cases + */ + if (info->attrs[NL80211_ATTR_FILS_CACHE_ID] && + info->attrs[NL80211_ATTR_PMK]) { + pmksa.cache_id = nla_data(info->attrs[NL80211_ATTR_FILS_CACHE_ID]); + } else if (!sae_offload_support && !owe_offload_support) { + return -EINVAL; + } + pmksa.ssid = nla_data(info->attrs[NL80211_ATTR_SSID]); + pmksa.ssid_len = nla_len(info->attrs[NL80211_ATTR_SSID]); + } else { + return -EINVAL; } - if (!rdev_ops) + if (dev->ieee80211_ptr->iftype != NL80211_IFTYPE_STATION && + dev->ieee80211_ptr->iftype != NL80211_IFTYPE_P2P_CLIENT && + !((dev->ieee80211_ptr->iftype == NL80211_IFTYPE_AP || + dev->ieee80211_ptr->iftype == NL80211_IFTYPE_P2P_GO) && + ap_pmksa_caching_support)) + return -EOPNOTSUPP; + + if (!rdev->ops->del_pmksa) return -EOPNOTSUPP; - return rdev_ops(&rdev->wiphy, dev, &pmksa); + return rdev_del_pmksa(rdev, dev, &pmksa); } static int nl80211_flush_pmksa(struct sk_buff *skb, struct genl_info *info) @@ -15847,7 +15923,7 @@ static int parse_tid_conf(struct cfg80211_registered_device *rdev, if (tid_conf->mask & ~mask) { NL_SET_ERR_MSG(extack, "unsupported TID configuration"); - return -ENOTSUPP; + return -EOPNOTSUPP; } return 0; @@ -16240,6 +16316,35 @@ static int nl80211_set_hw_timestamp(struct sk_buff *skb, return rdev_set_hw_timestamp(rdev, dev, &hwts); } +static int +nl80211_set_ttlm(struct sk_buff *skb, struct genl_info *info) +{ + struct cfg80211_ttlm_params params = {}; + struct cfg80211_registered_device *rdev = info->user_ptr[0]; + struct net_device *dev = info->user_ptr[1]; + struct wireless_dev *wdev = dev->ieee80211_ptr; + + if (wdev->iftype != NL80211_IFTYPE_STATION && + wdev->iftype != NL80211_IFTYPE_P2P_CLIENT) + return -EOPNOTSUPP; + + if (!wdev->connected) + return -ENOLINK; + + if (!info->attrs[NL80211_ATTR_MLO_TTLM_DLINK] || + !info->attrs[NL80211_ATTR_MLO_TTLM_ULINK]) + return -EINVAL; + + nla_memcpy(params.dlink, + info->attrs[NL80211_ATTR_MLO_TTLM_DLINK], + sizeof(params.dlink)); + nla_memcpy(params.ulink, + info->attrs[NL80211_ATTR_MLO_TTLM_ULINK], + sizeof(params.ulink)); + + return rdev_set_ttlm(rdev, dev, ¶ms); +} + #define NL80211_FLAG_NEED_WIPHY 0x01 #define NL80211_FLAG_NEED_NETDEV 0x02 #define NL80211_FLAG_NEED_RTNL 0x04 @@ -16928,7 +17033,7 @@ static const struct genl_small_ops nl80211_small_ops[] = { { .cmd = NL80211_CMD_SET_PMKSA, .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, - .doit = nl80211_setdel_pmksa, + .doit = nl80211_set_pmksa, .flags = GENL_UNS_ADMIN_PERM, .internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV_UP | NL80211_FLAG_CLEAR_SKB), @@ -16936,7 +17041,7 @@ static const struct genl_small_ops nl80211_small_ops[] = { { .cmd = NL80211_CMD_DEL_PMKSA, .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, - .doit = nl80211_setdel_pmksa, + .doit = nl80211_del_pmksa, .flags = GENL_UNS_ADMIN_PERM, .internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV_UP), }, @@ -17421,6 +17526,12 @@ static const struct genl_small_ops nl80211_small_ops[] = { .flags = GENL_UNS_ADMIN_PERM, .internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV_UP), }, + { + .cmd = NL80211_CMD_SET_TID_TO_LINK_MAPPING, + .doit = nl80211_set_ttlm, + .flags = GENL_UNS_ADMIN_PERM, + .internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV_UP), + }, }; static struct genl_family nl80211_fam __ro_after_init = { @@ -17752,21 +17863,29 @@ void nl80211_common_reg_change_event(enum nl80211_commands cmd_id, nlmsg_free(msg); } +struct nl80211_mlme_event { + enum nl80211_commands cmd; + const u8 *buf; + size_t buf_len; + int uapsd_queues; + const u8 *req_ies; + size_t req_ies_len; + bool reconnect; +}; + static void nl80211_send_mlme_event(struct cfg80211_registered_device *rdev, struct net_device *netdev, - const u8 *buf, size_t len, - enum nl80211_commands cmd, gfp_t gfp, - int uapsd_queues, const u8 *req_ies, - size_t req_ies_len, bool reconnect) + const struct nl80211_mlme_event *event, + gfp_t gfp) { struct sk_buff *msg; void *hdr; - msg = nlmsg_new(100 + len + req_ies_len, gfp); + msg = nlmsg_new(100 + event->buf_len + event->req_ies_len, gfp); if (!msg) return; - hdr = nl80211hdr_put(msg, 0, 0, 0, cmd); + hdr = nl80211hdr_put(msg, 0, 0, 0, event->cmd); if (!hdr) { nlmsg_free(msg); return; @@ -17774,22 +17893,24 @@ static void nl80211_send_mlme_event(struct cfg80211_registered_device *rdev, if (nla_put_u32(msg, NL80211_ATTR_WIPHY, rdev->wiphy_idx) || nla_put_u32(msg, NL80211_ATTR_IFINDEX, netdev->ifindex) || - nla_put(msg, NL80211_ATTR_FRAME, len, buf) || - (req_ies && - nla_put(msg, NL80211_ATTR_REQ_IE, req_ies_len, req_ies))) + nla_put(msg, NL80211_ATTR_FRAME, event->buf_len, event->buf) || + (event->req_ies && + nla_put(msg, NL80211_ATTR_REQ_IE, event->req_ies_len, + event->req_ies))) goto nla_put_failure; - if (reconnect && nla_put_flag(msg, NL80211_ATTR_RECONNECT_REQUESTED)) + if (event->reconnect && + nla_put_flag(msg, NL80211_ATTR_RECONNECT_REQUESTED)) goto nla_put_failure; - if (uapsd_queues >= 0) { + if (event->uapsd_queues >= 0) { struct nlattr *nla_wmm = nla_nest_start_noflag(msg, NL80211_ATTR_STA_WME); if (!nla_wmm) goto nla_put_failure; if (nla_put_u8(msg, NL80211_STA_WME_UAPSD_QUEUES, - uapsd_queues)) + event->uapsd_queues)) goto nla_put_failure; nla_nest_end(msg, nla_wmm); @@ -17809,37 +17930,60 @@ void nl80211_send_rx_auth(struct cfg80211_registered_device *rdev, struct net_device *netdev, const u8 *buf, size_t len, gfp_t gfp) { - nl80211_send_mlme_event(rdev, netdev, buf, len, - NL80211_CMD_AUTHENTICATE, gfp, -1, NULL, 0, - false); + struct nl80211_mlme_event event = { + .cmd = NL80211_CMD_AUTHENTICATE, + .buf = buf, + .buf_len = len, + .uapsd_queues = -1, + }; + + nl80211_send_mlme_event(rdev, netdev, &event, gfp); } void nl80211_send_rx_assoc(struct cfg80211_registered_device *rdev, struct net_device *netdev, - struct cfg80211_rx_assoc_resp_data *data) + const struct cfg80211_rx_assoc_resp_data *data) { - nl80211_send_mlme_event(rdev, netdev, data->buf, data->len, - NL80211_CMD_ASSOCIATE, GFP_KERNEL, - data->uapsd_queues, - data->req_ies, data->req_ies_len, false); + struct nl80211_mlme_event event = { + .cmd = NL80211_CMD_ASSOCIATE, + .buf = data->buf, + .buf_len = data->len, + .uapsd_queues = data->uapsd_queues, + .req_ies = data->req_ies, + .req_ies_len = data->req_ies_len, + }; + + nl80211_send_mlme_event(rdev, netdev, &event, GFP_KERNEL); } void nl80211_send_deauth(struct cfg80211_registered_device *rdev, struct net_device *netdev, const u8 *buf, size_t len, bool reconnect, gfp_t gfp) { - nl80211_send_mlme_event(rdev, netdev, buf, len, - NL80211_CMD_DEAUTHENTICATE, gfp, -1, NULL, 0, - reconnect); + struct nl80211_mlme_event event = { + .cmd = NL80211_CMD_DEAUTHENTICATE, + .buf = buf, + .buf_len = len, + .reconnect = reconnect, + .uapsd_queues = -1, + }; + + nl80211_send_mlme_event(rdev, netdev, &event, gfp); } void nl80211_send_disassoc(struct cfg80211_registered_device *rdev, struct net_device *netdev, const u8 *buf, size_t len, bool reconnect, gfp_t gfp) { - nl80211_send_mlme_event(rdev, netdev, buf, len, - NL80211_CMD_DISASSOCIATE, gfp, -1, NULL, 0, - reconnect); + struct nl80211_mlme_event event = { + .cmd = NL80211_CMD_DISASSOCIATE, + .buf = buf, + .buf_len = len, + .reconnect = reconnect, + .uapsd_queues = -1, + }; + + nl80211_send_mlme_event(rdev, netdev, &event, gfp); } void cfg80211_rx_unprot_mlme_mgmt(struct net_device *dev, const u8 *buf, @@ -17849,28 +17993,31 @@ void cfg80211_rx_unprot_mlme_mgmt(struct net_device *dev, const u8 *buf, struct wiphy *wiphy = wdev->wiphy; struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy); const struct ieee80211_mgmt *mgmt = (void *)buf; - u32 cmd; + struct nl80211_mlme_event event = { + .buf = buf, + .buf_len = len, + .uapsd_queues = -1, + }; if (WARN_ON(len < 2)) return; if (ieee80211_is_deauth(mgmt->frame_control)) { - cmd = NL80211_CMD_UNPROT_DEAUTHENTICATE; + event.cmd = NL80211_CMD_UNPROT_DEAUTHENTICATE; } else if (ieee80211_is_disassoc(mgmt->frame_control)) { - cmd = NL80211_CMD_UNPROT_DISASSOCIATE; + event.cmd = NL80211_CMD_UNPROT_DISASSOCIATE; } else if (ieee80211_is_beacon(mgmt->frame_control)) { if (wdev->unprot_beacon_reported && elapsed_jiffies_msecs(wdev->unprot_beacon_reported) < 10000) return; - cmd = NL80211_CMD_UNPROT_BEACON; + event.cmd = NL80211_CMD_UNPROT_BEACON; wdev->unprot_beacon_reported = jiffies; } else { return; } trace_cfg80211_rx_unprot_mlme_mgmt(dev, buf, len); - nl80211_send_mlme_event(rdev, dev, buf, len, cmd, GFP_ATOMIC, -1, - NULL, 0, false); + nl80211_send_mlme_event(rdev, dev, &event, GFP_ATOMIC); } EXPORT_SYMBOL(cfg80211_rx_unprot_mlme_mgmt); diff --git a/net/wireless/nl80211.h b/net/wireless/nl80211.h index aad40240d9cb29..6376f3a87f8abf 100644 --- a/net/wireless/nl80211.h +++ b/net/wireless/nl80211.h @@ -60,7 +60,7 @@ void nl80211_send_rx_auth(struct cfg80211_registered_device *rdev, const u8 *buf, size_t len, gfp_t gfp); void nl80211_send_rx_assoc(struct cfg80211_registered_device *rdev, struct net_device *netdev, - struct cfg80211_rx_assoc_resp_data *data); + const struct cfg80211_rx_assoc_resp_data *data); void nl80211_send_deauth(struct cfg80211_registered_device *rdev, struct net_device *netdev, const u8 *buf, size_t len, diff --git a/net/wireless/rdev-ops.h b/net/wireless/rdev-ops.h index 2214a90cf1012c..43897a5269b6a9 100644 --- a/net/wireless/rdev-ops.h +++ b/net/wireless/rdev-ops.h @@ -1046,7 +1046,7 @@ rdev_nan_change_conf(struct cfg80211_registered_device *rdev, ret = rdev->ops->nan_change_conf(&rdev->wiphy, wdev, conf, changes); else - ret = -ENOTSUPP; + ret = -EOPNOTSUPP; trace_rdev_return_int(&rdev->wiphy, ret); return ret; } @@ -1200,7 +1200,7 @@ rdev_start_radar_detection(struct cfg80211_registered_device *rdev, struct cfg80211_chan_def *chandef, u32 cac_time_ms) { - int ret = -ENOTSUPP; + int ret = -EOPNOTSUPP; trace_rdev_start_radar_detection(&rdev->wiphy, dev, chandef, cac_time_ms); @@ -1226,7 +1226,7 @@ rdev_set_mcast_rate(struct cfg80211_registered_device *rdev, struct net_device *dev, int mcast_rate[NUM_NL80211_BANDS]) { - int ret = -ENOTSUPP; + int ret = -EOPNOTSUPP; trace_rdev_set_mcast_rate(&rdev->wiphy, dev, mcast_rate); if (rdev->ops->set_mcast_rate) @@ -1239,7 +1239,7 @@ static inline int rdev_set_coalesce(struct cfg80211_registered_device *rdev, struct cfg80211_coalesce *coalesce) { - int ret = -ENOTSUPP; + int ret = -EOPNOTSUPP; trace_rdev_set_coalesce(&rdev->wiphy, coalesce); if (rdev->ops->set_coalesce) @@ -1524,4 +1524,22 @@ rdev_set_hw_timestamp(struct cfg80211_registered_device *rdev, return ret; } + +static inline int +rdev_set_ttlm(struct cfg80211_registered_device *rdev, + struct net_device *dev, + struct cfg80211_ttlm_params *params) +{ + struct wiphy *wiphy = &rdev->wiphy; + int ret; + + if (!rdev->ops->set_ttlm) + return -EOPNOTSUPP; + + trace_rdev_set_ttlm(wiphy, dev, params); + ret = rdev->ops->set_ttlm(wiphy, dev, params); + trace_rdev_return_int(wiphy, ret); + + return ret; +} #endif /* __CFG80211_RDEV_OPS */ diff --git a/net/wireless/scan.c b/net/wireless/scan.c index 9e5ccffd686841..3d260c99c34802 100644 --- a/net/wireless/scan.c +++ b/net/wireless/scan.c @@ -1535,12 +1535,13 @@ static bool cfg80211_bss_type_match(u16 capability, } /* Returned bss is reference counted and must be cleaned up appropriately. */ -struct cfg80211_bss *cfg80211_get_bss(struct wiphy *wiphy, - struct ieee80211_channel *channel, - const u8 *bssid, - const u8 *ssid, size_t ssid_len, - enum ieee80211_bss_type bss_type, - enum ieee80211_privacy privacy) +struct cfg80211_bss *__cfg80211_get_bss(struct wiphy *wiphy, + struct ieee80211_channel *channel, + const u8 *bssid, + const u8 *ssid, size_t ssid_len, + enum ieee80211_bss_type bss_type, + enum ieee80211_privacy privacy, + u32 use_for) { struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy); struct cfg80211_internal_bss *bss, *res = NULL; @@ -1565,6 +1566,8 @@ struct cfg80211_bss *cfg80211_get_bss(struct wiphy *wiphy, continue; if (!is_valid_ether_addr(bss->pub.bssid)) continue; + if ((bss->pub.use_for & use_for) != use_for) + continue; /* Don't get expired BSS structs */ if (time_after(now, bss->ts + IEEE80211_SCAN_RESULT_EXPIRE) && !atomic_read(&bss->hold)) @@ -1582,7 +1585,7 @@ struct cfg80211_bss *cfg80211_get_bss(struct wiphy *wiphy, trace_cfg80211_return_bss(&res->pub); return &res->pub; } -EXPORT_SYMBOL(cfg80211_get_bss); +EXPORT_SYMBOL(__cfg80211_get_bss); static void rb_insert_bss(struct cfg80211_registered_device *rdev, struct cfg80211_internal_bss *bss) @@ -1746,7 +1749,9 @@ cfg80211_update_known_bss(struct cfg80211_registered_device *rdev, new->pub.proberesp_ies); if (old) kfree_rcu((struct cfg80211_bss_ies *)old, rcu_head); - } else if (rcu_access_pointer(new->pub.beacon_ies)) { + } + + if (rcu_access_pointer(new->pub.beacon_ies)) { const struct cfg80211_bss_ies *old; if (known->pub.hidden_beacon_bss && @@ -1800,6 +1805,8 @@ cfg80211_update_known_bss(struct cfg80211_registered_device *rdev, ether_addr_copy(known->parent_bssid, new->parent_bssid); known->pub.max_bssid_indicator = new->pub.max_bssid_indicator; known->pub.bssid_index = new->pub.bssid_index; + known->pub.use_for &= new->pub.use_for; + known->pub.cannot_use_reasons = new->pub.cannot_use_reasons; return true; } @@ -2044,6 +2051,9 @@ struct cfg80211_inform_single_bss_data { struct cfg80211_bss *source_bss; u8 max_bssid_indicator; u8 bssid_index; + + u8 use_for; + u64 cannot_use_reasons; }; /* Returned bss is reference counted and must be cleaned up appropriately. */ @@ -2089,6 +2099,8 @@ cfg80211_inform_single_bss_data(struct wiphy *wiphy, tmp.ts_boottime = drv_data->boottime_ns; tmp.parent_tsf = drv_data->parent_tsf; ether_addr_copy(tmp.parent_bssid, drv_data->parent_bssid); + tmp.pub.use_for = data->use_for; + tmp.pub.cannot_use_reasons = data->cannot_use_reasons; if (data->bss_source != BSS_SOURCE_DIRECT) { tmp.pub.transmitted_bss = data->source_bss; @@ -2259,6 +2271,8 @@ cfg80211_parse_mbssid_data(struct wiphy *wiphy, .beacon_interval = tx_data->beacon_interval, .source_bss = source_bss, .bss_source = BSS_SOURCE_MBSSID, + .use_for = tx_data->use_for, + .cannot_use_reasons = tx_data->cannot_use_reasons, }; const u8 *mbssid_index_ie; const struct element *elem, *sub; @@ -2521,7 +2535,7 @@ cfg80211_defrag_mle(const struct element *mle, const u8 *ie, size_t ielen, return NULL; } -static bool +static u8 cfg80211_tbtt_info_for_mld_ap(const u8 *ie, size_t ielen, u8 mld_id, u8 link_id, const struct ieee80211_neighbor_ap_info **ap_info, const u8 **tbtt_info) @@ -2540,6 +2554,7 @@ cfg80211_tbtt_info_for_mld_ap(const u8 *ie, size_t ielen, u8 mld_id, u8 link_id, u16 params; u8 length, i, count, mld_params_offset; u8 type, lid; + u32 use_for; info = (void *)pos; count = u8_get_bits(info->tbtt_info_hdr, @@ -2549,20 +2564,22 @@ cfg80211_tbtt_info_for_mld_ap(const u8 *ie, size_t ielen, u8 mld_id, u8 link_id, pos += sizeof(*info); if (count * length > end - pos) - return false; + return 0; type = u8_get_bits(info->tbtt_info_hdr, IEEE80211_AP_INFO_TBTT_HDR_TYPE); - /* Only accept full TBTT information. NSTR mobile APs - * use the shortened version, but we ignore them here. - */ if (type == IEEE80211_TBTT_INFO_TYPE_TBTT && length >= offsetofend(struct ieee80211_tbtt_info_ge_11, mld_params)) { mld_params_offset = offsetof(struct ieee80211_tbtt_info_ge_11, mld_params); + use_for = NL80211_BSS_USE_FOR_ALL; + } else if (type == IEEE80211_TBTT_INFO_TYPE_MLD && + length >= sizeof(struct ieee80211_rnr_mld_params)) { + mld_params_offset = 0; + use_for = NL80211_BSS_USE_FOR_MLD_LINK; } else { pos += count * length; continue; @@ -2580,7 +2597,7 @@ cfg80211_tbtt_info_for_mld_ap(const u8 *ie, size_t ielen, u8 mld_id, u8 link_id, *ap_info = info; *tbtt_info = pos; - return true; + return use_for; } pos += length; @@ -2588,7 +2605,7 @@ cfg80211_tbtt_info_for_mld_ap(const u8 *ie, size_t ielen, u8 mld_id, u8 link_id, } } - return false; + return 0; } static void cfg80211_parse_ml_sta_data(struct wiphy *wiphy, @@ -2606,6 +2623,7 @@ static void cfg80211_parse_ml_sta_data(struct wiphy *wiphy, const struct element *elem; struct cfg80211_mle *mle; u16 control; + u8 ml_common_len; u8 *new_ie; struct cfg80211_bss *bss; int mld_id; @@ -2636,6 +2654,8 @@ static void cfg80211_parse_ml_sta_data(struct wiphy *wiphy, !(control & IEEE80211_MLC_BASIC_PRES_MLD_CAPA_OP)) return; + ml_common_len = ml_elem->variable[0]; + /* length + MLD MAC address + link ID info + BSS Params Change Count */ pos = ml_elem->variable + 1 + 6 + 1 + 1; @@ -2676,7 +2696,7 @@ static void cfg80211_parse_ml_sta_data(struct wiphy *wiphy, const u8 *profile; const u8 *tbtt_info; ssize_t profile_len; - u8 link_id; + u8 link_id, use_for; if (!ieee80211_mle_basic_sta_prof_size_ok((u8 *)mle->sta_prof[i], mle->sta_prof_len[i])) @@ -2718,9 +2738,11 @@ static void cfg80211_parse_ml_sta_data(struct wiphy *wiphy, profile_len -= 2; /* Find in RNR to look up channel information */ - if (!cfg80211_tbtt_info_for_mld_ap(tx_data->ie, tx_data->ielen, - mld_id, link_id, - &ap_info, &tbtt_info)) + use_for = cfg80211_tbtt_info_for_mld_ap(tx_data->ie, + tx_data->ielen, + mld_id, link_id, + &ap_info, &tbtt_info); + if (!use_for) continue; /* We could sanity check the BSSID is included */ @@ -2732,6 +2754,14 @@ static void cfg80211_parse_ml_sta_data(struct wiphy *wiphy, freq = ieee80211_channel_to_freq_khz(ap_info->channel, band); data.channel = ieee80211_get_channel_khz(wiphy, freq); + if (use_for == NL80211_BSS_USE_FOR_MLD_LINK && + !(wiphy->flags & WIPHY_FLAG_SUPPORTS_NSTR_NONPRIMARY)) { + use_for = 0; + data.cannot_use_reasons = + NL80211_BSS_CANNOT_USE_NSTR_NONPRIMARY; + } + data.use_for = use_for; + /* Generate new elements */ memset(new_ie, 0, IEEE80211_MAX_DATA_LEN); data.ie = new_ie; @@ -2742,6 +2772,34 @@ static void cfg80211_parse_ml_sta_data(struct wiphy *wiphy, if (!data.ielen) continue; + /* The generated elements do not contain: + * - Basic ML element + * - A TBTT entry in the RNR for the transmitting AP + * + * This information is needed both internally and in userspace + * as such, we should append it here. + */ + if (data.ielen + 3 + sizeof(*ml_elem) + ml_common_len > + IEEE80211_MAX_DATA_LEN) + continue; + + /* Copy the Basic Multi-Link element including the common + * information, and then fix up the link ID. + * Note that the ML element length has been verified and we + * also checked that it contains the link ID. + */ + new_ie[data.ielen++] = WLAN_EID_EXTENSION; + new_ie[data.ielen++] = 1 + sizeof(*ml_elem) + ml_common_len; + new_ie[data.ielen++] = WLAN_EID_EXT_EHT_MULTI_LINK; + memcpy(new_ie + data.ielen, ml_elem, + sizeof(*ml_elem) + ml_common_len); + + new_ie[data.ielen + sizeof(*ml_elem) + 1 + ETH_ALEN] = link_id; + + data.ielen += sizeof(*ml_elem) + ml_common_len; + + /* TODO: Add an RNR containing only the reporting AP */ + bss = cfg80211_inform_single_bss_data(wiphy, &data, gfp); if (!bss) break; @@ -2769,6 +2827,10 @@ cfg80211_inform_bss_data(struct wiphy *wiphy, .beacon_interval = beacon_interval, .ie = ie, .ielen = ielen, + .use_for = data->restrict_use ? + data->use_for : + NL80211_BSS_USE_FOR_ALL, + .cannot_use_reasons = data->cannot_use_reasons, }; struct cfg80211_bss *res; @@ -2899,6 +2961,10 @@ cfg80211_inform_single_bss_frame_data(struct wiphy *wiphy, tmp.pub.chains = data->chains; memcpy(tmp.pub.chain_signal, data->chain_signal, IEEE80211_MAX_CHAINS); ether_addr_copy(tmp.parent_bssid, data->parent_bssid); + tmp.pub.use_for = data->restrict_use ? + data->use_for : + NL80211_BSS_USE_FOR_ALL; + tmp.pub.cannot_use_reasons = data->cannot_use_reasons; signal_valid = data->chan == channel; spin_lock_bh(&rdev->bss_lock); @@ -2930,6 +2996,10 @@ cfg80211_inform_bss_frame_data(struct wiphy *wiphy, .ie = mgmt->u.probe_resp.variable, .ielen = len - offsetof(struct ieee80211_mgmt, u.probe_resp.variable), + .use_for = data->restrict_use ? + data->use_for : + NL80211_BSS_USE_FOR_ALL, + .cannot_use_reasons = data->cannot_use_reasons, }; struct cfg80211_bss *res; diff --git a/net/wireless/trace.h b/net/wireless/trace.h index 30cd1bd58aac06..1f374c8a17a505 100644 --- a/net/wireless/trace.h +++ b/net/wireless/trace.h @@ -2928,7 +2928,7 @@ DEFINE_EVENT(netdev_evt_only, cfg80211_send_rx_auth, TRACE_EVENT(cfg80211_send_rx_assoc, TP_PROTO(struct net_device *netdev, - struct cfg80211_rx_assoc_resp_data *data), + const struct cfg80211_rx_assoc_resp_data *data), TP_ARGS(netdev, data), TP_STRUCT__entry( NETDEV_ENTRY @@ -3979,6 +3979,26 @@ TRACE_EVENT(cfg80211_links_removed, __entry->link_mask) ); +TRACE_EVENT(rdev_set_ttlm, + TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, + struct cfg80211_ttlm_params *params), + TP_ARGS(wiphy, netdev, params), + TP_STRUCT__entry( + WIPHY_ENTRY + NETDEV_ENTRY + __array(u8, dlink, sizeof(u16) * 8) + __array(u8, ulink, sizeof(u16) * 8) + ), + TP_fast_assign( + WIPHY_ASSIGN; + NETDEV_ASSIGN; + memcpy(__entry->dlink, params->dlink, sizeof(params->dlink)); + memcpy(__entry->ulink, params->ulink, sizeof(params->ulink)); + ), + TP_printk(WIPHY_PR_FMT ", " NETDEV_PR_FMT, + WIPHY_PR_ARG, NETDEV_PR_ARG) +); + #endif /* !__RDEV_OPS_TRACE || TRACE_HEADER_MULTI_READ */ #undef TRACE_INCLUDE_PATH diff --git a/net/xdp/xsk_buff_pool.c b/net/xdp/xsk_buff_pool.c index 4f6f538a546277..28711cc44ced21 100644 --- a/net/xdp/xsk_buff_pool.c +++ b/net/xdp/xsk_buff_pool.c @@ -125,6 +125,18 @@ void xp_set_rxq_info(struct xsk_buff_pool *pool, struct xdp_rxq_info *rxq) } EXPORT_SYMBOL(xp_set_rxq_info); +void xp_fill_cb(struct xsk_buff_pool *pool, struct xsk_cb_desc *desc) +{ + u32 i; + + for (i = 0; i < pool->heads_cnt; i++) { + struct xdp_buff_xsk *xskb = &pool->heads[i]; + + memcpy(xskb->cb + desc->off, desc->src, desc->bytes); + } +} +EXPORT_SYMBOL(xp_fill_cb); + static void xp_disable_drv_zc(struct xsk_buff_pool *pool) { struct netdev_bpf bpf; diff --git a/net/xfrm/Makefile b/net/xfrm/Makefile index cd47f88921f597..547cec77ba03cb 100644 --- a/net/xfrm/Makefile +++ b/net/xfrm/Makefile @@ -21,3 +21,4 @@ obj-$(CONFIG_XFRM_USER_COMPAT) += xfrm_compat.o obj-$(CONFIG_XFRM_IPCOMP) += xfrm_ipcomp.o obj-$(CONFIG_XFRM_INTERFACE) += xfrm_interface.o obj-$(CONFIG_XFRM_ESPINTCP) += espintcp.o +obj-$(CONFIG_DEBUG_INFO_BTF) += xfrm_state_bpf.o diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c index c13dc3ef791072..1b7e7515972779 100644 --- a/net/xfrm/xfrm_policy.c +++ b/net/xfrm/xfrm_policy.c @@ -4218,6 +4218,8 @@ void __init xfrm_init(void) #ifdef CONFIG_XFRM_ESPINTCP espintcp_init(); #endif + + register_xfrm_state_bpf(); } #ifdef CONFIG_AUDITSYSCALL diff --git a/net/xfrm/xfrm_state_bpf.c b/net/xfrm/xfrm_state_bpf.c new file mode 100644 index 00000000000000..9e20d4a377f7eb --- /dev/null +++ b/net/xfrm/xfrm_state_bpf.c @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Unstable XFRM state BPF helpers. + * + * Note that it is allowed to break compatibility for these functions since the + * interface they are exposed through to BPF programs is explicitly unstable. + */ + +#include +#include +#include +#include +#include + +/* bpf_xfrm_state_opts - Options for XFRM state lookup helpers + * + * Members: + * @error - Out parameter, set for any errors encountered + * Values: + * -EINVAL - netns_id is less than -1 + * -EINVAL - opts__sz isn't BPF_XFRM_STATE_OPTS_SZ + * -ENONET - No network namespace found for netns_id + * -ENOENT - No xfrm_state found + * @netns_id - Specify the network namespace for lookup + * Values: + * BPF_F_CURRENT_NETNS (-1) + * Use namespace associated with ctx + * [0, S32_MAX] + * Network Namespace ID + * @mark - XFRM mark to match on + * @daddr - Destination address to match on + * @spi - Security parameter index to match on + * @proto - IP protocol to match on (eg. IPPROTO_ESP) + * @family - Protocol family to match on (AF_INET/AF_INET6) + */ +struct bpf_xfrm_state_opts { + s32 error; + s32 netns_id; + u32 mark; + xfrm_address_t daddr; + __be32 spi; + u8 proto; + u16 family; +}; + +enum { + BPF_XFRM_STATE_OPTS_SZ = sizeof(struct bpf_xfrm_state_opts), +}; + +__bpf_kfunc_start_defs(); + +/* bpf_xdp_get_xfrm_state - Get XFRM state + * + * A `struct xfrm_state *`, if found, must be released with a corresponding + * bpf_xdp_xfrm_state_release. + * + * Parameters: + * @ctx - Pointer to ctx (xdp_md) in XDP program + * Cannot be NULL + * @opts - Options for lookup (documented above) + * Cannot be NULL + * @opts__sz - Length of the bpf_xfrm_state_opts structure + * Must be BPF_XFRM_STATE_OPTS_SZ + */ +__bpf_kfunc struct xfrm_state * +bpf_xdp_get_xfrm_state(struct xdp_md *ctx, struct bpf_xfrm_state_opts *opts, u32 opts__sz) +{ + struct xdp_buff *xdp = (struct xdp_buff *)ctx; + struct net *net = dev_net(xdp->rxq->dev); + struct xfrm_state *x; + + if (!opts || opts__sz < sizeof(opts->error)) + return NULL; + + if (opts__sz != BPF_XFRM_STATE_OPTS_SZ) { + opts->error = -EINVAL; + return NULL; + } + + if (unlikely(opts->netns_id < BPF_F_CURRENT_NETNS)) { + opts->error = -EINVAL; + return NULL; + } + + if (opts->netns_id >= 0) { + net = get_net_ns_by_id(net, opts->netns_id); + if (unlikely(!net)) { + opts->error = -ENONET; + return NULL; + } + } + + x = xfrm_state_lookup(net, opts->mark, &opts->daddr, opts->spi, + opts->proto, opts->family); + + if (opts->netns_id >= 0) + put_net(net); + if (!x) + opts->error = -ENOENT; + + return x; +} + +/* bpf_xdp_xfrm_state_release - Release acquired xfrm_state object + * + * This must be invoked for referenced PTR_TO_BTF_ID, and the verifier rejects + * the program if any references remain in the program in all of the explored + * states. + * + * Parameters: + * @x - Pointer to referenced xfrm_state object, obtained using + * bpf_xdp_get_xfrm_state. + */ +__bpf_kfunc void bpf_xdp_xfrm_state_release(struct xfrm_state *x) +{ + xfrm_state_put(x); +} + +__bpf_kfunc_end_defs(); + +BTF_SET8_START(xfrm_state_kfunc_set) +BTF_ID_FLAGS(func, bpf_xdp_get_xfrm_state, KF_RET_NULL | KF_ACQUIRE) +BTF_ID_FLAGS(func, bpf_xdp_xfrm_state_release, KF_RELEASE) +BTF_SET8_END(xfrm_state_kfunc_set) + +static const struct btf_kfunc_id_set xfrm_state_xdp_kfunc_set = { + .owner = THIS_MODULE, + .set = &xfrm_state_kfunc_set, +}; + +int __init register_xfrm_state_bpf(void) +{ + return register_btf_kfunc_id_set(BPF_PROG_TYPE_XDP, + &xfrm_state_xdp_kfunc_set); +} diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 85f013ed4ca4cf..eaf01df7d97a71 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -8,6 +8,9 @@ #include #include +#include +#include +#include #include #include #include diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index e6aff80b521fd5..7ac39874aeac36 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -14,6 +14,7 @@ #![no_std] #![feature(allocator_api)] #![feature(coerce_unsized)] +#![feature(const_maybe_uninit_zeroed)] #![feature(dispatch_from_dyn)] #![feature(new_uninit)] #![feature(offset_of)] @@ -38,6 +39,8 @@ pub mod init; pub mod ioctl; #[cfg(CONFIG_KUNIT)] pub mod kunit; +#[cfg(CONFIG_NET)] +pub mod net; pub mod prelude; pub mod print; mod static_assert; diff --git a/rust/kernel/net.rs b/rust/kernel/net.rs new file mode 100644 index 00000000000000..fe415cb369d3ac --- /dev/null +++ b/rust/kernel/net.rs @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Networking. + +#[cfg(CONFIG_RUST_PHYLIB_ABSTRACTIONS)] +pub mod phy; diff --git a/rust/kernel/net/phy.rs b/rust/kernel/net/phy.rs new file mode 100644 index 00000000000000..e457b3c7cb2fc5 --- /dev/null +++ b/rust/kernel/net/phy.rs @@ -0,0 +1,901 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2023 FUJITA Tomonori + +//! Network PHY device. +//! +//! C headers: [`include/linux/phy.h`](../../../../../../../include/linux/phy.h). + +use crate::{bindings, error::*, prelude::*, str::CStr, types::Opaque}; + +use core::marker::PhantomData; + +/// PHY state machine states. +/// +/// Corresponds to the kernel's [`enum phy_state`]. +/// +/// Some of PHY drivers access to the state of PHY's software state machine. +/// +/// [`enum phy_state`]: ../../../../../../../include/linux/phy.h +#[derive(PartialEq, Eq)] +pub enum DeviceState { + /// PHY device and driver are not ready for anything. + Down, + /// PHY is ready to send and receive packets. + Ready, + /// PHY is up, but no polling or interrupts are done. + Halted, + /// PHY is up, but is in an error state. + Error, + /// PHY and attached device are ready to do work. + Up, + /// PHY is currently running. + Running, + /// PHY is up, but not currently plugged in. + NoLink, + /// PHY is performing a cable test. + CableTest, +} + +/// A mode of Ethernet communication. +/// +/// PHY drivers get duplex information from hardware and update the current state. +pub enum DuplexMode { + /// PHY is in full-duplex mode. + Full, + /// PHY is in half-duplex mode. + Half, + /// PHY is in unknown duplex mode. + Unknown, +} + +/// An instance of a PHY device. +/// +/// Wraps the kernel's [`struct phy_device`]. +/// +/// A [`Device`] instance is created when a callback in [`Driver`] is executed. A PHY driver +/// executes [`Driver`]'s methods during the callback. +/// +/// # Invariants +/// +/// Referencing a `phy_device` using this struct asserts that you are in +/// a context where all methods defined on this struct are safe to call. +/// +/// [`struct phy_device`]: ../../../../../../../include/linux/phy.h +// During the calls to most functions in [`Driver`], the C side (`PHYLIB`) holds a lock that is +// unique for every instance of [`Device`]. `PHYLIB` uses a different serialization technique for +// [`Driver::resume`] and [`Driver::suspend`]: `PHYLIB` updates `phy_device`'s state with +// the lock held, thus guaranteeing that [`Driver::resume`] has exclusive access to the instance. +// [`Driver::resume`] and [`Driver::suspend`] also are called where only one thread can access +// to the instance. +#[repr(transparent)] +pub struct Device(Opaque); + +impl Device { + /// Creates a new [`Device`] instance from a raw pointer. + /// + /// # Safety + /// + /// For the duration of 'a, the pointer must point at a valid `phy_device`, + /// and the caller must be in a context where all methods defined on this struct + /// are safe to call. + unsafe fn from_raw<'a>(ptr: *mut bindings::phy_device) -> &'a mut Self { + // CAST: `Self` is a `repr(transparent)` wrapper around `bindings::phy_device`. + let ptr = ptr.cast::(); + // SAFETY: by the function requirements the pointer is valid and we have unique access for + // the duration of `'a`. + unsafe { &mut *ptr } + } + + /// Gets the id of the PHY. + pub fn phy_id(&self) -> u32 { + let phydev = self.0.get(); + // SAFETY: The struct invariant ensures that we may access + // this field without additional synchronization. + unsafe { (*phydev).phy_id } + } + + /// Gets the state of PHY state machine states. + pub fn state(&self) -> DeviceState { + let phydev = self.0.get(); + // SAFETY: The struct invariant ensures that we may access + // this field without additional synchronization. + let state = unsafe { (*phydev).state }; + // TODO: this conversion code will be replaced with automatically generated code by bindgen + // when it becomes possible. + match state { + bindings::phy_state_PHY_DOWN => DeviceState::Down, + bindings::phy_state_PHY_READY => DeviceState::Ready, + bindings::phy_state_PHY_HALTED => DeviceState::Halted, + bindings::phy_state_PHY_ERROR => DeviceState::Error, + bindings::phy_state_PHY_UP => DeviceState::Up, + bindings::phy_state_PHY_RUNNING => DeviceState::Running, + bindings::phy_state_PHY_NOLINK => DeviceState::NoLink, + bindings::phy_state_PHY_CABLETEST => DeviceState::CableTest, + _ => DeviceState::Error, + } + } + + /// Gets the current link state. + /// + /// It returns true if the link is up. + pub fn is_link_up(&self) -> bool { + const LINK_IS_UP: u64 = 1; + // TODO: the code to access to the bit field will be replaced with automatically + // generated code by bindgen when it becomes possible. + // SAFETY: The struct invariant ensures that we may access + // this field without additional synchronization. + let bit_field = unsafe { &(*self.0.get())._bitfield_1 }; + bit_field.get(14, 1) == LINK_IS_UP + } + + /// Gets the current auto-negotiation configuration. + /// + /// It returns true if auto-negotiation is enabled. + pub fn is_autoneg_enabled(&self) -> bool { + // TODO: the code to access to the bit field will be replaced with automatically + // generated code by bindgen when it becomes possible. + // SAFETY: The struct invariant ensures that we may access + // this field without additional synchronization. + let bit_field = unsafe { &(*self.0.get())._bitfield_1 }; + bit_field.get(13, 1) == bindings::AUTONEG_ENABLE as u64 + } + + /// Gets the current auto-negotiation state. + /// + /// It returns true if auto-negotiation is completed. + pub fn is_autoneg_completed(&self) -> bool { + const AUTONEG_COMPLETED: u64 = 1; + // TODO: the code to access to the bit field will be replaced with automatically + // generated code by bindgen when it becomes possible. + // SAFETY: The struct invariant ensures that we may access + // this field without additional synchronization. + let bit_field = unsafe { &(*self.0.get())._bitfield_1 }; + bit_field.get(15, 1) == AUTONEG_COMPLETED + } + + /// Sets the speed of the PHY. + pub fn set_speed(&mut self, speed: u32) { + let phydev = self.0.get(); + // SAFETY: The struct invariant ensures that we may access + // this field without additional synchronization. + unsafe { (*phydev).speed = speed as i32 }; + } + + /// Sets duplex mode. + pub fn set_duplex(&mut self, mode: DuplexMode) { + let phydev = self.0.get(); + let v = match mode { + DuplexMode::Full => bindings::DUPLEX_FULL as i32, + DuplexMode::Half => bindings::DUPLEX_HALF as i32, + DuplexMode::Unknown => bindings::DUPLEX_UNKNOWN as i32, + }; + // SAFETY: The struct invariant ensures that we may access + // this field without additional synchronization. + unsafe { (*phydev).duplex = v }; + } + + /// Reads a given C22 PHY register. + // This function reads a hardware register and updates the stats so takes `&mut self`. + pub fn read(&mut self, regnum: u16) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call, open code of `phy_read()` with a valid `phy_device` pointer + // `phydev`. + let ret = unsafe { + bindings::mdiobus_read((*phydev).mdio.bus, (*phydev).mdio.addr, regnum.into()) + }; + if ret < 0 { + Err(Error::from_errno(ret)) + } else { + Ok(ret as u16) + } + } + + /// Writes a given C22 PHY register. + pub fn write(&mut self, regnum: u16, val: u16) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call, open code of `phy_write()` with a valid `phy_device` pointer + // `phydev`. + to_result(unsafe { + bindings::mdiobus_write((*phydev).mdio.bus, (*phydev).mdio.addr, regnum.into(), val) + }) + } + + /// Reads a paged register. + pub fn read_paged(&mut self, page: u16, regnum: u16) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + let ret = unsafe { bindings::phy_read_paged(phydev, page.into(), regnum.into()) }; + if ret < 0 { + Err(Error::from_errno(ret)) + } else { + Ok(ret as u16) + } + } + + /// Resolves the advertisements into PHY settings. + pub fn resolve_aneg_linkmode(&mut self) { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + unsafe { bindings::phy_resolve_aneg_linkmode(phydev) }; + } + + /// Executes software reset the PHY via `BMCR_RESET` bit. + pub fn genphy_soft_reset(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::genphy_soft_reset(phydev) }) + } + + /// Initializes the PHY. + pub fn init_hw(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::phy_init_hw(phydev) }) + } + + /// Starts auto-negotiation. + pub fn start_aneg(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::_phy_start_aneg(phydev) }) + } + + /// Resumes the PHY via `BMCR_PDOWN` bit. + pub fn genphy_resume(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::genphy_resume(phydev) }) + } + + /// Suspends the PHY via `BMCR_PDOWN` bit. + pub fn genphy_suspend(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::genphy_suspend(phydev) }) + } + + /// Checks the link status and updates current link state. + pub fn genphy_read_status(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + let ret = unsafe { bindings::genphy_read_status(phydev) }; + if ret < 0 { + Err(Error::from_errno(ret)) + } else { + Ok(ret as u16) + } + } + + /// Updates the link status. + pub fn genphy_update_link(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::genphy_update_link(phydev) }) + } + + /// Reads link partner ability. + pub fn genphy_read_lpa(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::genphy_read_lpa(phydev) }) + } + + /// Reads PHY abilities. + pub fn genphy_read_abilities(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::genphy_read_abilities(phydev) }) + } +} + +/// Defines certain other features this PHY supports (like interrupts). +/// +/// These flag values are used in [`Driver::FLAGS`]. +pub mod flags { + /// PHY is internal. + pub const IS_INTERNAL: u32 = bindings::PHY_IS_INTERNAL; + /// PHY needs to be reset after the refclk is enabled. + pub const RST_AFTER_CLK_EN: u32 = bindings::PHY_RST_AFTER_CLK_EN; + /// Polling is used to detect PHY status changes. + pub const POLL_CABLE_TEST: u32 = bindings::PHY_POLL_CABLE_TEST; + /// Don't suspend. + pub const ALWAYS_CALL_SUSPEND: u32 = bindings::PHY_ALWAYS_CALL_SUSPEND; +} + +/// An adapter for the registration of a PHY driver. +struct Adapter { + _p: PhantomData, +} + +impl Adapter { + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn soft_reset_callback( + phydev: *mut bindings::phy_device, + ) -> core::ffi::c_int { + from_result(|| { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::soft_reset(dev)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn get_features_callback( + phydev: *mut bindings::phy_device, + ) -> core::ffi::c_int { + from_result(|| { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::get_features(dev)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn suspend_callback(phydev: *mut bindings::phy_device) -> core::ffi::c_int { + from_result(|| { + // SAFETY: The C core code ensures that the accessors on + // `Device` are okay to call even though `phy_device->lock` + // might not be held. + let dev = unsafe { Device::from_raw(phydev) }; + T::suspend(dev)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn resume_callback(phydev: *mut bindings::phy_device) -> core::ffi::c_int { + from_result(|| { + // SAFETY: The C core code ensures that the accessors on + // `Device` are okay to call even though `phy_device->lock` + // might not be held. + let dev = unsafe { Device::from_raw(phydev) }; + T::resume(dev)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn config_aneg_callback( + phydev: *mut bindings::phy_device, + ) -> core::ffi::c_int { + from_result(|| { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::config_aneg(dev)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn read_status_callback( + phydev: *mut bindings::phy_device, + ) -> core::ffi::c_int { + from_result(|| { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::read_status(dev)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn match_phy_device_callback( + phydev: *mut bindings::phy_device, + ) -> core::ffi::c_int { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::match_phy_device(dev) as i32 + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn read_mmd_callback( + phydev: *mut bindings::phy_device, + devnum: i32, + regnum: u16, + ) -> i32 { + from_result(|| { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + // CAST: the C side verifies devnum < 32. + let ret = T::read_mmd(dev, devnum as u8, regnum)?; + Ok(ret.into()) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn write_mmd_callback( + phydev: *mut bindings::phy_device, + devnum: i32, + regnum: u16, + val: u16, + ) -> i32 { + from_result(|| { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::write_mmd(dev, devnum as u8, regnum, val)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn link_change_notify_callback(phydev: *mut bindings::phy_device) { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::link_change_notify(dev); + } +} + +/// Driver structure for a particular PHY type. +/// +/// Wraps the kernel's [`struct phy_driver`]. +/// This is used to register a driver for a particular PHY type with the kernel. +/// +/// # Invariants +/// +/// `self.0` is always in a valid state. +/// +/// [`struct phy_driver`]: ../../../../../../../include/linux/phy.h +#[repr(transparent)] +pub struct DriverVTable(Opaque); + +// SAFETY: `DriverVTable` doesn't expose any &self method to access internal data, so it's safe to +// share `&DriverVTable` across execution context boundries. +unsafe impl Sync for DriverVTable {} + +/// Creates a [`DriverVTable`] instance from [`Driver`]. +/// +/// This is used by [`module_phy_driver`] macro to create a static array of `phy_driver`. +/// +/// [`module_phy_driver`]: crate::module_phy_driver +pub const fn create_phy_driver() -> DriverVTable { + // INVARIANT: All the fields of `struct phy_driver` are initialized properly. + DriverVTable(Opaque::new(bindings::phy_driver { + name: T::NAME.as_char_ptr().cast_mut(), + flags: T::FLAGS, + phy_id: T::PHY_DEVICE_ID.id, + phy_id_mask: T::PHY_DEVICE_ID.mask_as_int(), + soft_reset: if T::HAS_SOFT_RESET { + Some(Adapter::::soft_reset_callback) + } else { + None + }, + get_features: if T::HAS_GET_FEATURES { + Some(Adapter::::get_features_callback) + } else { + None + }, + match_phy_device: if T::HAS_MATCH_PHY_DEVICE { + Some(Adapter::::match_phy_device_callback) + } else { + None + }, + suspend: if T::HAS_SUSPEND { + Some(Adapter::::suspend_callback) + } else { + None + }, + resume: if T::HAS_RESUME { + Some(Adapter::::resume_callback) + } else { + None + }, + config_aneg: if T::HAS_CONFIG_ANEG { + Some(Adapter::::config_aneg_callback) + } else { + None + }, + read_status: if T::HAS_READ_STATUS { + Some(Adapter::::read_status_callback) + } else { + None + }, + read_mmd: if T::HAS_READ_MMD { + Some(Adapter::::read_mmd_callback) + } else { + None + }, + write_mmd: if T::HAS_WRITE_MMD { + Some(Adapter::::write_mmd_callback) + } else { + None + }, + link_change_notify: if T::HAS_LINK_CHANGE_NOTIFY { + Some(Adapter::::link_change_notify_callback) + } else { + None + }, + // SAFETY: The rest is zeroed out to initialize `struct phy_driver`, + // sets `Option<&F>` to be `None`. + ..unsafe { core::mem::MaybeUninit::::zeroed().assume_init() } + })) +} + +/// Driver implementation for a particular PHY type. +/// +/// This trait is used to create a [`DriverVTable`]. +#[vtable] +pub trait Driver { + /// Defines certain other features this PHY supports. + /// It is a combination of the flags in the [`flags`] module. + const FLAGS: u32 = 0; + + /// The friendly name of this PHY type. + const NAME: &'static CStr; + + /// This driver only works for PHYs with IDs which match this field. + /// The default id and mask are zero. + const PHY_DEVICE_ID: DeviceId = DeviceId::new_with_custom_mask(0, 0); + + /// Issues a PHY software reset. + fn soft_reset(_dev: &mut Device) -> Result { + Err(code::ENOTSUPP) + } + + /// Probes the hardware to determine what abilities it has. + fn get_features(_dev: &mut Device) -> Result { + Err(code::ENOTSUPP) + } + + /// Returns true if this is a suitable driver for the given phydev. + /// If not implemented, matching is based on [`Driver::PHY_DEVICE_ID`]. + fn match_phy_device(_dev: &Device) -> bool { + false + } + + /// Configures the advertisement and resets auto-negotiation + /// if auto-negotiation is enabled. + fn config_aneg(_dev: &mut Device) -> Result { + Err(code::ENOTSUPP) + } + + /// Determines the negotiated speed and duplex. + fn read_status(_dev: &mut Device) -> Result { + Err(code::ENOTSUPP) + } + + /// Suspends the hardware, saving state if needed. + fn suspend(_dev: &mut Device) -> Result { + Err(code::ENOTSUPP) + } + + /// Resumes the hardware, restoring state if needed. + fn resume(_dev: &mut Device) -> Result { + Err(code::ENOTSUPP) + } + + /// Overrides the default MMD read function for reading a MMD register. + fn read_mmd(_dev: &mut Device, _devnum: u8, _regnum: u16) -> Result { + Err(code::ENOTSUPP) + } + + /// Overrides the default MMD write function for writing a MMD register. + fn write_mmd(_dev: &mut Device, _devnum: u8, _regnum: u16, _val: u16) -> Result { + Err(code::ENOTSUPP) + } + + /// Callback for notification of link change. + fn link_change_notify(_dev: &mut Device) {} +} + +/// Registration structure for PHY drivers. +/// +/// Registers [`DriverVTable`] instances with the kernel. They will be unregistered when dropped. +/// +/// # Invariants +/// +/// The `drivers` slice are currently registered to the kernel via `phy_drivers_register`. +pub struct Registration { + drivers: Pin<&'static mut [DriverVTable]>, +} + +impl Registration { + /// Registers a PHY driver. + pub fn register( + module: &'static crate::ThisModule, + drivers: Pin<&'static mut [DriverVTable]>, + ) -> Result { + if drivers.is_empty() { + return Err(code::EINVAL); + } + // SAFETY: The type invariants of [`DriverVTable`] ensure that all elements of + // the `drivers` slice are initialized properly. `drivers` will not be moved. + // So it's just an FFI call. + to_result(unsafe { + bindings::phy_drivers_register(drivers[0].0.get(), drivers.len().try_into()?, module.0) + })?; + // INVARIANT: The `drivers` slice is successfully registered to the kernel via `phy_drivers_register`. + Ok(Registration { drivers }) + } +} + +impl Drop for Registration { + fn drop(&mut self) { + // SAFETY: The type invariants guarantee that `self.drivers` is valid. + // So it's just an FFI call. + unsafe { + bindings::phy_drivers_unregister(self.drivers[0].0.get(), self.drivers.len() as i32) + }; + } +} + +/// An identifier for PHY devices on an MDIO/MII bus. +/// +/// Represents the kernel's `struct mdio_device_id`. This is used to find an appropriate +/// PHY driver. +pub struct DeviceId { + id: u32, + mask: DeviceMask, +} + +impl DeviceId { + /// Creates a new instance with the exact match mask. + pub const fn new_with_exact_mask(id: u32) -> Self { + DeviceId { + id, + mask: DeviceMask::Exact, + } + } + + /// Creates a new instance with the model match mask. + pub const fn new_with_model_mask(id: u32) -> Self { + DeviceId { + id, + mask: DeviceMask::Model, + } + } + + /// Creates a new instance with the vendor match mask. + pub const fn new_with_vendor_mask(id: u32) -> Self { + DeviceId { + id, + mask: DeviceMask::Vendor, + } + } + + /// Creates a new instance with a custom match mask. + pub const fn new_with_custom_mask(id: u32, mask: u32) -> Self { + DeviceId { + id, + mask: DeviceMask::Custom(mask), + } + } + + /// Creates a new instance from [`Driver`]. + pub const fn new_with_driver() -> Self { + T::PHY_DEVICE_ID + } + + /// Get a `mask` as u32. + pub const fn mask_as_int(&self) -> u32 { + self.mask.as_int() + } + + // macro use only + #[doc(hidden)] + pub const fn mdio_device_id(&self) -> bindings::mdio_device_id { + bindings::mdio_device_id { + phy_id: self.id, + phy_id_mask: self.mask.as_int(), + } + } +} + +enum DeviceMask { + Exact, + Model, + Vendor, + Custom(u32), +} + +impl DeviceMask { + const MASK_EXACT: u32 = !0; + const MASK_MODEL: u32 = !0 << 4; + const MASK_VENDOR: u32 = !0 << 10; + + const fn as_int(&self) -> u32 { + match self { + DeviceMask::Exact => Self::MASK_EXACT, + DeviceMask::Model => Self::MASK_MODEL, + DeviceMask::Vendor => Self::MASK_VENDOR, + DeviceMask::Custom(mask) => *mask, + } + } +} + +/// Declares a kernel module for PHYs drivers. +/// +/// This creates a static array of kernel's `struct phy_driver` and registers it. +/// This also corresponds to the kernel's `MODULE_DEVICE_TABLE` macro, which embeds the information +/// for module loading into the module binary file. Every driver needs an entry in `device_table`. +/// +/// # Examples +/// +/// ``` +/// # mod module_phy_driver_sample { +/// use kernel::c_str; +/// use kernel::net::phy::{self, DeviceId}; +/// use kernel::prelude::*; +/// +/// kernel::module_phy_driver! { +/// drivers: [PhySample], +/// device_table: [ +/// DeviceId::new_with_driver::() +/// ], +/// name: "rust_sample_phy", +/// author: "Rust for Linux Contributors", +/// description: "Rust sample PHYs driver", +/// license: "GPL", +/// } +/// +/// struct PhySample; +/// +/// #[vtable] +/// impl phy::Driver for PhySample { +/// const NAME: &'static CStr = c_str!("PhySample"); +/// const PHY_DEVICE_ID: phy::DeviceId = phy::DeviceId::new_with_exact_mask(0x00000001); +/// } +/// # } +/// ``` +/// +/// This expands to the following code: +/// +/// ```ignore +/// use kernel::c_str; +/// use kernel::net::phy::{self, DeviceId}; +/// use kernel::prelude::*; +/// +/// struct Module { +/// _reg: ::kernel::net::phy::Registration, +/// } +/// +/// module! { +/// type: Module, +/// name: "rust_sample_phy", +/// author: "Rust for Linux Contributors", +/// description: "Rust sample PHYs driver", +/// license: "GPL", +/// } +/// +/// struct PhySample; +/// +/// #[vtable] +/// impl phy::Driver for PhySample { +/// const NAME: &'static CStr = c_str!("PhySample"); +/// const PHY_DEVICE_ID: phy::DeviceId = phy::DeviceId::new_with_exact_mask(0x00000001); +/// } +/// +/// const _: () = { +/// static mut DRIVERS: [::kernel::net::phy::DriverVTable; 1] = +/// [::kernel::net::phy::create_phy_driver::()]; +/// +/// impl ::kernel::Module for Module { +/// fn init(module: &'static ThisModule) -> Result { +/// let drivers = unsafe { &mut DRIVERS }; +/// let mut reg = ::kernel::net::phy::Registration::register( +/// module, +/// ::core::pin::Pin::static_mut(drivers), +/// )?; +/// Ok(Module { _reg: reg }) +/// } +/// } +/// }; +/// +/// #[cfg(MODULE)] +/// #[no_mangle] +/// static __mod_mdio__phydev_device_table: [::kernel::bindings::mdio_device_id; 2] = [ +/// ::kernel::bindings::mdio_device_id { +/// phy_id: 0x00000001, +/// phy_id_mask: 0xffffffff, +/// }, +/// ::kernel::bindings::mdio_device_id { +/// phy_id: 0, +/// phy_id_mask: 0, +/// }, +/// ]; +/// ``` +#[macro_export] +macro_rules! module_phy_driver { + (@replace_expr $_t:tt $sub:expr) => {$sub}; + + (@count_devices $($x:expr),*) => { + 0usize $(+ $crate::module_phy_driver!(@replace_expr $x 1usize))* + }; + + (@device_table [$($dev:expr),+]) => { + // SAFETY: C will not read off the end of this constant since the last element is zero. + #[cfg(MODULE)] + #[no_mangle] + static __mod_mdio__phydev_device_table: [$crate::bindings::mdio_device_id; + $crate::module_phy_driver!(@count_devices $($dev),+) + 1] = [ + $($dev.mdio_device_id()),+, + $crate::bindings::mdio_device_id { + phy_id: 0, + phy_id_mask: 0 + } + ]; + }; + + (drivers: [$($driver:ident),+ $(,)?], device_table: [$($dev:expr),+ $(,)?], $($f:tt)*) => { + struct Module { + _reg: $crate::net::phy::Registration, + } + + $crate::prelude::module! { + type: Module, + $($f)* + } + + const _: () = { + static mut DRIVERS: [$crate::net::phy::DriverVTable; + $crate::module_phy_driver!(@count_devices $($driver),+)] = + [$($crate::net::phy::create_phy_driver::<$driver>()),+]; + + impl $crate::Module for Module { + fn init(module: &'static ThisModule) -> Result { + // SAFETY: The anonymous constant guarantees that nobody else can access + // the `DRIVERS` static. The array is used only in the C side. + let drivers = unsafe { &mut DRIVERS }; + let mut reg = $crate::net::phy::Registration::register( + module, + ::core::pin::Pin::static_mut(drivers), + )?; + Ok(Module { _reg: reg }) + } + } + }; + + $crate::module_phy_driver!(@device_table [$($dev),+]); + } +} diff --git a/rust/uapi/uapi_helper.h b/rust/uapi/uapi_helper.h index 301f5207f0238d..08f5e9334c9e83 100644 --- a/rust/uapi/uapi_helper.h +++ b/rust/uapi/uapi_helper.h @@ -7,3 +7,5 @@ */ #include +#include +#include diff --git a/security/security.c b/security/security.c index dcb3e7014f9bdd..088a79c35c2625 100644 --- a/security/security.c +++ b/security/security.c @@ -5167,29 +5167,87 @@ int security_bpf_prog(struct bpf_prog *prog) } /** - * security_bpf_map_alloc() - Allocate a bpf map LSM blob - * @map: bpf map + * security_bpf_map_create() - Check if BPF map creation is allowed + * @map: BPF map object + * @attr: BPF syscall attributes used to create BPF map + * @token: BPF token used to grant user access + * + * Do a check when the kernel creates a new BPF map. This is also the + * point where LSM blob is allocated for LSMs that need them. + * + * Return: Returns 0 on success, error on failure. + */ +int security_bpf_map_create(struct bpf_map *map, union bpf_attr *attr, + struct bpf_token *token) +{ + return call_int_hook(bpf_map_create, 0, map, attr, token); +} + +/** + * security_bpf_prog_load() - Check if loading of BPF program is allowed + * @prog: BPF program object + * @attr: BPF syscall attributes used to create BPF program + * @token: BPF token used to grant user access to BPF subsystem * - * Initialize the security field inside bpf map. + * Perform an access control check when the kernel loads a BPF program and + * allocates associated BPF program object. This hook is also responsible for + * allocating any required LSM state for the BPF program. * * Return: Returns 0 on success, error on failure. */ -int security_bpf_map_alloc(struct bpf_map *map) +int security_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr, + struct bpf_token *token) { - return call_int_hook(bpf_map_alloc_security, 0, map); + return call_int_hook(bpf_prog_load, 0, prog, attr, token); } /** - * security_bpf_prog_alloc() - Allocate a bpf program LSM blob - * @aux: bpf program aux info struct + * security_bpf_token_create() - Check if creating of BPF token is allowed + * @token: BPF token object + * @attr: BPF syscall attributes used to create BPF token + * @path: path pointing to BPF FS mount point from which BPF token is created * - * Initialize the security field inside bpf program. + * Do a check when the kernel instantiates a new BPF token object from BPF FS + * instance. This is also the point where LSM blob can be allocated for LSMs. * * Return: Returns 0 on success, error on failure. */ -int security_bpf_prog_alloc(struct bpf_prog_aux *aux) +int security_bpf_token_create(struct bpf_token *token, union bpf_attr *attr, + struct path *path) { - return call_int_hook(bpf_prog_alloc_security, 0, aux); + return call_int_hook(bpf_token_create, 0, token, attr, path); +} + +/** + * security_bpf_token_cmd() - Check if BPF token is allowed to delegate + * requested BPF syscall command + * @token: BPF token object + * @cmd: BPF syscall command requested to be delegated by BPF token + * + * Do a check when the kernel decides whether provided BPF token should allow + * delegation of requested BPF syscall command. + * + * Return: Returns 0 on success, error on failure. + */ +int security_bpf_token_cmd(const struct bpf_token *token, enum bpf_cmd cmd) +{ + return call_int_hook(bpf_token_cmd, 0, token, cmd); +} + +/** + * security_bpf_token_capable() - Check if BPF token is allowed to delegate + * requested BPF-related capability + * @token: BPF token object + * @cap: capabilities requested to be delegated by BPF token + * + * Do a check when the kernel decides whether provided BPF token should allow + * delegation of requested BPF-related capabilities. + * + * Return: Returns 0 on success, error on failure. + */ +int security_bpf_token_capable(const struct bpf_token *token, int cap) +{ + return call_int_hook(bpf_token_capable, 0, token, cap); } /** @@ -5200,18 +5258,29 @@ int security_bpf_prog_alloc(struct bpf_prog_aux *aux) */ void security_bpf_map_free(struct bpf_map *map) { - call_void_hook(bpf_map_free_security, map); + call_void_hook(bpf_map_free, map); +} + +/** + * security_bpf_prog_free() - Free a BPF program's LSM blob + * @prog: BPF program struct + * + * Clean up the security information stored inside BPF program. + */ +void security_bpf_prog_free(struct bpf_prog *prog) +{ + call_void_hook(bpf_prog_free, prog); } /** - * security_bpf_prog_free() - Free a bpf program's LSM blob - * @aux: bpf program aux info struct + * security_bpf_token_free() - Free a BPF token's LSM blob + * @token: BPF token struct * - * Clean up the security information stored inside bpf prog. + * Clean up the security information stored inside BPF token. */ -void security_bpf_prog_free(struct bpf_prog_aux *aux) +void security_bpf_token_free(struct bpf_token *token) { - call_void_hook(bpf_prog_free_security, aux); + call_void_hook(bpf_token_free, token); } #endif /* CONFIG_BPF_SYSCALL */ diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index feda711c6b7b84..1501e95366a1a4 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -6783,7 +6783,8 @@ static int selinux_bpf_prog(struct bpf_prog *prog) BPF__PROG_RUN, NULL); } -static int selinux_bpf_map_alloc(struct bpf_map *map) +static int selinux_bpf_map_create(struct bpf_map *map, union bpf_attr *attr, + struct bpf_token *token) { struct bpf_security_struct *bpfsec; @@ -6805,7 +6806,8 @@ static void selinux_bpf_map_free(struct bpf_map *map) kfree(bpfsec); } -static int selinux_bpf_prog_alloc(struct bpf_prog_aux *aux) +static int selinux_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr, + struct bpf_token *token) { struct bpf_security_struct *bpfsec; @@ -6814,16 +6816,39 @@ static int selinux_bpf_prog_alloc(struct bpf_prog_aux *aux) return -ENOMEM; bpfsec->sid = current_sid(); - aux->security = bpfsec; + prog->aux->security = bpfsec; return 0; } -static void selinux_bpf_prog_free(struct bpf_prog_aux *aux) +static void selinux_bpf_prog_free(struct bpf_prog *prog) { - struct bpf_security_struct *bpfsec = aux->security; + struct bpf_security_struct *bpfsec = prog->aux->security; - aux->security = NULL; + prog->aux->security = NULL; + kfree(bpfsec); +} + +static int selinux_bpf_token_create(struct bpf_token *token, union bpf_attr *attr, + struct path *path) +{ + struct bpf_security_struct *bpfsec; + + bpfsec = kzalloc(sizeof(*bpfsec), GFP_KERNEL); + if (!bpfsec) + return -ENOMEM; + + bpfsec->sid = current_sid(); + token->security = bpfsec; + + return 0; +} + +static void selinux_bpf_token_free(struct bpf_token *token) +{ + struct bpf_security_struct *bpfsec = token->security; + + token->security = NULL; kfree(bpfsec); } #endif @@ -7179,8 +7204,9 @@ static struct security_hook_list selinux_hooks[] __ro_after_init = { LSM_HOOK_INIT(bpf, selinux_bpf), LSM_HOOK_INIT(bpf_map, selinux_bpf_map), LSM_HOOK_INIT(bpf_prog, selinux_bpf_prog), - LSM_HOOK_INIT(bpf_map_free_security, selinux_bpf_map_free), - LSM_HOOK_INIT(bpf_prog_free_security, selinux_bpf_prog_free), + LSM_HOOK_INIT(bpf_map_free, selinux_bpf_map_free), + LSM_HOOK_INIT(bpf_prog_free, selinux_bpf_prog_free), + LSM_HOOK_INIT(bpf_token_free, selinux_bpf_token_free), #endif #ifdef CONFIG_PERF_EVENTS @@ -7237,8 +7263,9 @@ static struct security_hook_list selinux_hooks[] __ro_after_init = { LSM_HOOK_INIT(audit_rule_init, selinux_audit_rule_init), #endif #ifdef CONFIG_BPF_SYSCALL - LSM_HOOK_INIT(bpf_map_alloc_security, selinux_bpf_map_alloc), - LSM_HOOK_INIT(bpf_prog_alloc_security, selinux_bpf_prog_alloc), + LSM_HOOK_INIT(bpf_map_create, selinux_bpf_map_create), + LSM_HOOK_INIT(bpf_prog_load, selinux_bpf_prog_load), + LSM_HOOK_INIT(bpf_token_create, selinux_bpf_token_create), #endif #ifdef CONFIG_PERF_EVENTS LSM_HOOK_INIT(perf_event_alloc, selinux_perf_event_alloc), diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index e88746ba7d2198..e0545201b55f6d 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -847,6 +847,36 @@ union bpf_iter_link_info { * Returns zero on success. On error, -1 is returned and *errno* * is set appropriately. * + * BPF_TOKEN_CREATE + * Description + * Create BPF token with embedded information about what + * BPF-related functionality it allows: + * - a set of allowed bpf() syscall commands; + * - a set of allowed BPF map types to be created with + * BPF_MAP_CREATE command, if BPF_MAP_CREATE itself is allowed; + * - a set of allowed BPF program types and BPF program attach + * types to be loaded with BPF_PROG_LOAD command, if + * BPF_PROG_LOAD itself is allowed. + * + * BPF token is created (derived) from an instance of BPF FS, + * assuming it has necessary delegation mount options specified. + * This BPF token can be passed as an extra parameter to various + * bpf() syscall commands to grant BPF subsystem functionality to + * unprivileged processes. + * + * When created, BPF token is "associated" with the owning + * user namespace of BPF FS instance (super block) that it was + * derived from, and subsequent BPF operations performed with + * BPF token would be performing capabilities checks (i.e., + * CAP_BPF, CAP_PERFMON, CAP_NET_ADMIN, CAP_SYS_ADMIN) within + * that user namespace. Without BPF token, such capabilities + * have to be granted in init user namespace, making bpf() + * syscall incompatible with user namespace, for the most part. + * + * Return + * A new file descriptor (a nonnegative integer), or -1 if an + * error occurred (in which case, *errno* is set appropriately). + * * NOTES * eBPF objects (maps and programs) can be shared between processes. * @@ -901,6 +931,8 @@ enum bpf_cmd { BPF_ITER_CREATE, BPF_LINK_DETACH, BPF_PROG_BIND_MAP, + BPF_TOKEN_CREATE, + __MAX_BPF_CMD, }; enum bpf_map_type { @@ -951,6 +983,7 @@ enum bpf_map_type { BPF_MAP_TYPE_BLOOM_FILTER, BPF_MAP_TYPE_USER_RINGBUF, BPF_MAP_TYPE_CGRP_STORAGE, + __MAX_BPF_MAP_TYPE }; /* Note that tracing related programs such as @@ -995,6 +1028,7 @@ enum bpf_prog_type { BPF_PROG_TYPE_SK_LOOKUP, BPF_PROG_TYPE_SYSCALL, /* a program that can execute syscalls */ BPF_PROG_TYPE_NETFILTER, + __MAX_BPF_PROG_TYPE }; enum bpf_attach_type { @@ -1074,9 +1108,11 @@ enum bpf_link_type { BPF_LINK_TYPE_TCX = 11, BPF_LINK_TYPE_UPROBE_MULTI = 12, BPF_LINK_TYPE_NETKIT = 13, - MAX_BPF_LINK_TYPE, + __MAX_BPF_LINK_TYPE, }; +#define MAX_BPF_LINK_TYPE __MAX_BPF_LINK_TYPE + enum bpf_perf_event_type { BPF_PERF_EVENT_UNSPEC = 0, BPF_PERF_EVENT_UPROBE = 1, @@ -1401,6 +1437,7 @@ union bpf_attr { * to using 5 hash functions). */ __u64 map_extra; + __u32 map_token_fd; }; struct { /* anonymous struct used by BPF_MAP_*_ELEM commands */ @@ -1470,6 +1507,7 @@ union bpf_attr { * truncated), or smaller (if log buffer wasn't filled completely). */ __u32 log_true_size; + __u32 prog_token_fd; }; struct { /* anonymous struct used by BPF_OBJ_* commands */ @@ -1582,6 +1620,7 @@ union bpf_attr { * truncated), or smaller (if log buffer wasn't filled completely). */ __u32 btf_log_true_size; + __u32 btf_token_fd; }; struct { @@ -1712,6 +1751,11 @@ union bpf_attr { __u32 flags; /* extra flags */ } prog_bind_map; + struct { /* struct used by BPF_TOKEN_CREATE command */ + __u32 flags; + __u32 bpffs_fd; + } token_create; + } __attribute__((aligned(8))); /* The description below is an attempt at providing documentation to eBPF diff --git a/tools/include/uapi/linux/netdev.h b/tools/include/uapi/linux/netdev.h index 424c5e28f49519..93cb411adf72e4 100644 --- a/tools/include/uapi/linux/netdev.h +++ b/tools/include/uapi/linux/netdev.h @@ -44,10 +44,13 @@ enum netdev_xdp_act { * timestamp via bpf_xdp_metadata_rx_timestamp(). * @NETDEV_XDP_RX_METADATA_HASH: Device is capable of exposing receive packet * hash via bpf_xdp_metadata_rx_hash(). + * @NETDEV_XDP_RX_METADATA_VLAN_TAG: Device is capable of exposing receive + * packet VLAN tag via bpf_xdp_metadata_rx_vlan_tag(). */ enum netdev_xdp_rx_metadata { NETDEV_XDP_RX_METADATA_TIMESTAMP = 1, NETDEV_XDP_RX_METADATA_HASH = 2, + NETDEV_XDP_RX_METADATA_VLAN_TAG = 4, }; /** diff --git a/tools/lib/bpf/Build b/tools/lib/bpf/Build index 2d0c282c8588b5..b6619199a7066a 100644 --- a/tools/lib/bpf/Build +++ b/tools/lib/bpf/Build @@ -1,4 +1,4 @@ libbpf-y := libbpf.o bpf.o nlattr.o btf.o libbpf_errno.o str_error.o \ netlink.o bpf_prog_linfo.o libbpf_probes.o hashmap.o \ btf_dump.o ringbuf.o strset.o linker.o gen_loader.o relo_core.o \ - usdt.o zip.o elf.o + usdt.o zip.o elf.o features.o diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c index 9dc9625651dcff..0ad8e532b3cfc4 100644 --- a/tools/lib/bpf/bpf.c +++ b/tools/lib/bpf/bpf.c @@ -103,7 +103,7 @@ int sys_bpf_prog_load(union bpf_attr *attr, unsigned int size, int attempts) * [0] https://lore.kernel.org/bpf/20201201215900.3569844-1-guro@fb.com/ * [1] d05512618056 ("bpf: Add bpf_ktime_get_coarse_ns helper") */ -int probe_memcg_account(void) +int probe_memcg_account(int token_fd) { const size_t attr_sz = offsetofend(union bpf_attr, attach_btf_obj_fd); struct bpf_insn insns[] = { @@ -120,6 +120,7 @@ int probe_memcg_account(void) attr.insns = ptr_to_u64(insns); attr.insn_cnt = insn_cnt; attr.license = ptr_to_u64("GPL"); + attr.prog_token_fd = token_fd; prog_fd = sys_bpf_fd(BPF_PROG_LOAD, &attr, attr_sz); if (prog_fd >= 0) { @@ -146,7 +147,7 @@ int bump_rlimit_memlock(void) struct rlimit rlim; /* if kernel supports memcg-based accounting, skip bumping RLIMIT_MEMLOCK */ - if (memlock_bumped || kernel_supports(NULL, FEAT_MEMCG_ACCOUNT)) + if (memlock_bumped || feat_supported(NULL, FEAT_MEMCG_ACCOUNT)) return 0; memlock_bumped = true; @@ -169,7 +170,7 @@ int bpf_map_create(enum bpf_map_type map_type, __u32 max_entries, const struct bpf_map_create_opts *opts) { - const size_t attr_sz = offsetofend(union bpf_attr, map_extra); + const size_t attr_sz = offsetofend(union bpf_attr, map_token_fd); union bpf_attr attr; int fd; @@ -181,7 +182,7 @@ int bpf_map_create(enum bpf_map_type map_type, return libbpf_err(-EINVAL); attr.map_type = map_type; - if (map_name && kernel_supports(NULL, FEAT_PROG_NAME)) + if (map_name && feat_supported(NULL, FEAT_PROG_NAME)) libbpf_strlcpy(attr.map_name, map_name, sizeof(attr.map_name)); attr.key_size = key_size; attr.value_size = value_size; @@ -198,6 +199,8 @@ int bpf_map_create(enum bpf_map_type map_type, attr.numa_node = OPTS_GET(opts, numa_node, 0); attr.map_ifindex = OPTS_GET(opts, map_ifindex, 0); + attr.map_token_fd = OPTS_GET(opts, token_fd, 0); + fd = sys_bpf_fd(BPF_MAP_CREATE, &attr, attr_sz); return libbpf_err_errno(fd); } @@ -232,7 +235,7 @@ int bpf_prog_load(enum bpf_prog_type prog_type, const struct bpf_insn *insns, size_t insn_cnt, struct bpf_prog_load_opts *opts) { - const size_t attr_sz = offsetofend(union bpf_attr, log_true_size); + const size_t attr_sz = offsetofend(union bpf_attr, prog_token_fd); void *finfo = NULL, *linfo = NULL; const char *func_info, *line_info; __u32 log_size, log_level, attach_prog_fd, attach_btf_obj_fd; @@ -261,8 +264,9 @@ int bpf_prog_load(enum bpf_prog_type prog_type, attr.prog_flags = OPTS_GET(opts, prog_flags, 0); attr.prog_ifindex = OPTS_GET(opts, prog_ifindex, 0); attr.kern_version = OPTS_GET(opts, kern_version, 0); + attr.prog_token_fd = OPTS_GET(opts, token_fd, 0); - if (prog_name && kernel_supports(NULL, FEAT_PROG_NAME)) + if (prog_name && feat_supported(NULL, FEAT_PROG_NAME)) libbpf_strlcpy(attr.prog_name, prog_name, sizeof(attr.prog_name)); attr.license = ptr_to_u64(license); @@ -1182,7 +1186,7 @@ int bpf_raw_tracepoint_open(const char *name, int prog_fd) int bpf_btf_load(const void *btf_data, size_t btf_size, struct bpf_btf_load_opts *opts) { - const size_t attr_sz = offsetofend(union bpf_attr, btf_log_true_size); + const size_t attr_sz = offsetofend(union bpf_attr, btf_token_fd); union bpf_attr attr; char *log_buf; size_t log_size; @@ -1207,6 +1211,8 @@ int bpf_btf_load(const void *btf_data, size_t btf_size, struct bpf_btf_load_opts attr.btf = ptr_to_u64(btf_data); attr.btf_size = btf_size; + attr.btf_token_fd = OPTS_GET(opts, token_fd, 0); + /* log_level == 0 and log_buf != NULL means "try loading without * log_buf, but retry with log_buf and log_level=1 on error", which is * consistent across low-level and high-level BTF and program loading @@ -1287,3 +1293,20 @@ int bpf_prog_bind_map(int prog_fd, int map_fd, ret = sys_bpf(BPF_PROG_BIND_MAP, &attr, attr_sz); return libbpf_err_errno(ret); } + +int bpf_token_create(int bpffs_fd, struct bpf_token_create_opts *opts) +{ + const size_t attr_sz = offsetofend(union bpf_attr, token_create); + union bpf_attr attr; + int fd; + + if (!OPTS_VALID(opts, bpf_token_create_opts)) + return libbpf_err(-EINVAL); + + memset(&attr, 0, attr_sz); + attr.token_create.bpffs_fd = bpffs_fd; + attr.token_create.flags = OPTS_GET(opts, flags, 0); + + fd = sys_bpf_fd(BPF_TOKEN_CREATE, &attr, attr_sz); + return libbpf_err_errno(fd); +} diff --git a/tools/lib/bpf/bpf.h b/tools/lib/bpf/bpf.h index d0f53772bdc02a..991b86bfe7e412 100644 --- a/tools/lib/bpf/bpf.h +++ b/tools/lib/bpf/bpf.h @@ -51,8 +51,11 @@ struct bpf_map_create_opts { __u32 numa_node; __u32 map_ifindex; + + __u32 token_fd; + size_t :0; }; -#define bpf_map_create_opts__last_field map_ifindex +#define bpf_map_create_opts__last_field token_fd LIBBPF_API int bpf_map_create(enum bpf_map_type map_type, const char *map_name, @@ -102,9 +105,10 @@ struct bpf_prog_load_opts { * If kernel doesn't support this feature, log_size is left unchanged. */ __u32 log_true_size; + __u32 token_fd; size_t :0; }; -#define bpf_prog_load_opts__last_field log_true_size +#define bpf_prog_load_opts__last_field token_fd LIBBPF_API int bpf_prog_load(enum bpf_prog_type prog_type, const char *prog_name, const char *license, @@ -130,9 +134,10 @@ struct bpf_btf_load_opts { * If kernel doesn't support this feature, log_size is left unchanged. */ __u32 log_true_size; + __u32 token_fd; size_t :0; }; -#define bpf_btf_load_opts__last_field log_true_size +#define bpf_btf_load_opts__last_field token_fd LIBBPF_API int bpf_btf_load(const void *btf_data, size_t btf_size, struct bpf_btf_load_opts *opts); @@ -640,6 +645,30 @@ struct bpf_test_run_opts { LIBBPF_API int bpf_prog_test_run_opts(int prog_fd, struct bpf_test_run_opts *opts); +struct bpf_token_create_opts { + size_t sz; /* size of this struct for forward/backward compatibility */ + __u32 flags; + size_t :0; +}; +#define bpf_token_create_opts__last_field flags + +/** + * @brief **bpf_token_create()** creates a new instance of BPF token derived + * from specified BPF FS mount point. + * + * BPF token created with this API can be passed to bpf() syscall for + * commands like BPF_PROG_LOAD, BPF_MAP_CREATE, etc. + * + * @param bpffs_fd FD for BPF FS instance from which to derive a BPF token + * instance. + * @param opts optional BPF token creation options, can be NULL + * + * @return BPF token FD > 0, on success; negative error code, otherwise (errno + * is also set to the error code) + */ +LIBBPF_API int bpf_token_create(int bpffs_fd, + struct bpf_token_create_opts *opts); + #ifdef __cplusplus } /* extern "C" */ #endif diff --git a/tools/lib/bpf/bpf_core_read.h b/tools/lib/bpf/bpf_core_read.h index 1ac57bb7ac55f7..7325a12692a33e 100644 --- a/tools/lib/bpf/bpf_core_read.h +++ b/tools/lib/bpf/bpf_core_read.h @@ -111,6 +111,38 @@ enum bpf_enum_value_kind { val; \ }) +/* + * Write to a bitfield, identified by s->field. + * This is the inverse of BPF_CORE_WRITE_BITFIELD(). + */ +#define BPF_CORE_WRITE_BITFIELD(s, field, new_val) ({ \ + void *p = (void *)s + __CORE_RELO(s, field, BYTE_OFFSET); \ + unsigned int byte_size = __CORE_RELO(s, field, BYTE_SIZE); \ + unsigned int lshift = __CORE_RELO(s, field, LSHIFT_U64); \ + unsigned int rshift = __CORE_RELO(s, field, RSHIFT_U64); \ + unsigned long long mask, val, nval = new_val; \ + unsigned int rpad = rshift - lshift; \ + \ + asm volatile("" : "+r"(p)); \ + \ + switch (byte_size) { \ + case 1: val = *(unsigned char *)p; break; \ + case 2: val = *(unsigned short *)p; break; \ + case 4: val = *(unsigned int *)p; break; \ + case 8: val = *(unsigned long long *)p; break; \ + } \ + \ + mask = (~0ULL << rshift) >> lshift; \ + val = (val & ~mask) | ((nval << rpad) & mask); \ + \ + switch (byte_size) { \ + case 1: *(unsigned char *)p = val; break; \ + case 2: *(unsigned short *)p = val; break; \ + case 4: *(unsigned int *)p = val; break; \ + case 8: *(unsigned long long *)p = val; break; \ + } \ +}) + #define ___bpf_field_ref1(field) (field) #define ___bpf_field_ref2(type, field) (((typeof(type) *)0)->field) #define ___bpf_field_ref(args...) \ diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c index ee95fd379d4d8a..63033c3343205f 100644 --- a/tools/lib/bpf/btf.c +++ b/tools/lib/bpf/btf.c @@ -1317,7 +1317,9 @@ struct btf *btf__parse_split(const char *path, struct btf *base_btf) static void *btf_get_raw_data(const struct btf *btf, __u32 *size, bool swap_endian); -int btf_load_into_kernel(struct btf *btf, char *log_buf, size_t log_sz, __u32 log_level) +int btf_load_into_kernel(struct btf *btf, + char *log_buf, size_t log_sz, __u32 log_level, + int token_fd) { LIBBPF_OPTS(bpf_btf_load_opts, opts); __u32 buf_sz = 0, raw_size; @@ -1367,6 +1369,7 @@ int btf_load_into_kernel(struct btf *btf, char *log_buf, size_t log_sz, __u32 lo opts.log_level = log_level; } + opts.token_fd = token_fd; btf->fd = bpf_btf_load(raw_data, raw_size, &opts); if (btf->fd < 0) { /* time to turn on verbose mode and try again */ @@ -1394,7 +1397,7 @@ int btf_load_into_kernel(struct btf *btf, char *log_buf, size_t log_sz, __u32 lo int btf__load_into_kernel(struct btf *btf) { - return btf_load_into_kernel(btf, NULL, 0, 0); + return btf_load_into_kernel(btf, NULL, 0, 0, 0); } int btf__fd(const struct btf *btf) diff --git a/tools/lib/bpf/elf.c b/tools/lib/bpf/elf.c index b02faec748a51b..c92e02394159e1 100644 --- a/tools/lib/bpf/elf.c +++ b/tools/lib/bpf/elf.c @@ -11,8 +11,6 @@ #include "libbpf_internal.h" #include "str_error.h" -#define STRERR_BUFSIZE 128 - /* A SHT_GNU_versym section holds 16-bit words. This bit is set if * the symbol is hidden and can only be seen when referenced using an * explicit version number. This is a GNU extension. diff --git a/tools/lib/bpf/features.c b/tools/lib/bpf/features.c new file mode 100644 index 00000000000000..ce98a334be21ee --- /dev/null +++ b/tools/lib/bpf/features.c @@ -0,0 +1,478 @@ +// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) +/* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */ +#include +#include +#include "bpf.h" +#include "libbpf.h" +#include "libbpf_common.h" +#include "libbpf_internal.h" +#include "str_error.h" + +static inline __u64 ptr_to_u64(const void *ptr) +{ + return (__u64)(unsigned long)ptr; +} + +static int probe_fd(int fd) +{ + if (fd >= 0) + close(fd); + return fd >= 0; +} + +static int probe_kern_prog_name(int token_fd) +{ + const size_t attr_sz = offsetofend(union bpf_attr, prog_name); + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + union bpf_attr attr; + int ret; + + memset(&attr, 0, attr_sz); + attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; + attr.license = ptr_to_u64("GPL"); + attr.insns = ptr_to_u64(insns); + attr.insn_cnt = (__u32)ARRAY_SIZE(insns); + attr.prog_token_fd = token_fd; + libbpf_strlcpy(attr.prog_name, "libbpf_nametest", sizeof(attr.prog_name)); + + /* make sure loading with name works */ + ret = sys_bpf_prog_load(&attr, attr_sz, PROG_LOAD_ATTEMPTS); + return probe_fd(ret); +} + +static int probe_kern_global_data(int token_fd) +{ + char *cp, errmsg[STRERR_BUFSIZE]; + struct bpf_insn insns[] = { + BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16), + BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + LIBBPF_OPTS(bpf_map_create_opts, map_opts, .token_fd = token_fd); + LIBBPF_OPTS(bpf_prog_load_opts, prog_opts, .token_fd = token_fd); + int ret, map, insn_cnt = ARRAY_SIZE(insns); + + map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_global", sizeof(int), 32, 1, &map_opts); + if (map < 0) { + ret = -errno; + cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); + pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n", + __func__, cp, -ret); + return ret; + } + + insns[0].imm = map; + + ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &prog_opts); + close(map); + return probe_fd(ret); +} + +static int probe_kern_btf(int token_fd) +{ + static const char strs[] = "\0int"; + __u32 types[] = { + /* int */ + BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), + }; + + return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), + strs, sizeof(strs), token_fd)); +} + +static int probe_kern_btf_func(int token_fd) +{ + static const char strs[] = "\0int\0x\0a"; + /* void x(int a) {} */ + __u32 types[] = { + /* int */ + BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + /* FUNC_PROTO */ /* [2] */ + BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), + BTF_PARAM_ENC(7, 1), + /* FUNC x */ /* [3] */ + BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2), + }; + + return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), + strs, sizeof(strs), token_fd)); +} + +static int probe_kern_btf_func_global(int token_fd) +{ + static const char strs[] = "\0int\0x\0a"; + /* static void x(int a) {} */ + __u32 types[] = { + /* int */ + BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + /* FUNC_PROTO */ /* [2] */ + BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), + BTF_PARAM_ENC(7, 1), + /* FUNC x BTF_FUNC_GLOBAL */ /* [3] */ + BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 2), + }; + + return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), + strs, sizeof(strs), token_fd)); +} + +static int probe_kern_btf_datasec(int token_fd) +{ + static const char strs[] = "\0x\0.data"; + /* static int a; */ + __u32 types[] = { + /* int */ + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + /* VAR x */ /* [2] */ + BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1), + BTF_VAR_STATIC, + /* DATASEC val */ /* [3] */ + BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4), + BTF_VAR_SECINFO_ENC(2, 0, 4), + }; + + return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), + strs, sizeof(strs), token_fd)); +} + +static int probe_kern_btf_float(int token_fd) +{ + static const char strs[] = "\0float"; + __u32 types[] = { + /* float */ + BTF_TYPE_FLOAT_ENC(1, 4), + }; + + return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), + strs, sizeof(strs), token_fd)); +} + +static int probe_kern_btf_decl_tag(int token_fd) +{ + static const char strs[] = "\0tag"; + __u32 types[] = { + /* int */ + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + /* VAR x */ /* [2] */ + BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1), + BTF_VAR_STATIC, + /* attr */ + BTF_TYPE_DECL_TAG_ENC(1, 2, -1), + }; + + return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), + strs, sizeof(strs), token_fd)); +} + +static int probe_kern_btf_type_tag(int token_fd) +{ + static const char strs[] = "\0tag"; + __u32 types[] = { + /* int */ + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + /* attr */ + BTF_TYPE_TYPE_TAG_ENC(1, 1), /* [2] */ + /* ptr */ + BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_PTR, 0, 0), 2), /* [3] */ + }; + + return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), + strs, sizeof(strs), token_fd)); +} + +static int probe_kern_array_mmap(int token_fd) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts, + .map_flags = BPF_F_MMAPABLE, + .token_fd = token_fd, + ); + int fd; + + fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_mmap", sizeof(int), sizeof(int), 1, &opts); + return probe_fd(fd); +} + +static int probe_kern_exp_attach_type(int token_fd) +{ + LIBBPF_OPTS(bpf_prog_load_opts, opts, + .expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE, + .token_fd = token_fd, + ); + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + int fd, insn_cnt = ARRAY_SIZE(insns); + + /* use any valid combination of program type and (optional) + * non-zero expected attach type (i.e., not a BPF_CGROUP_INET_INGRESS) + * to see if kernel supports expected_attach_type field for + * BPF_PROG_LOAD command + */ + fd = bpf_prog_load(BPF_PROG_TYPE_CGROUP_SOCK, NULL, "GPL", insns, insn_cnt, &opts); + return probe_fd(fd); +} + +static int probe_kern_probe_read_kernel(int token_fd) +{ + LIBBPF_OPTS(bpf_prog_load_opts, opts, .token_fd = token_fd); + struct bpf_insn insns[] = { + BPF_MOV64_REG(BPF_REG_1, BPF_REG_10), /* r1 = r10 (fp) */ + BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8), /* r1 += -8 */ + BPF_MOV64_IMM(BPF_REG_2, 8), /* r2 = 8 */ + BPF_MOV64_IMM(BPF_REG_3, 0), /* r3 = 0 */ + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_probe_read_kernel), + BPF_EXIT_INSN(), + }; + int fd, insn_cnt = ARRAY_SIZE(insns); + + fd = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, &opts); + return probe_fd(fd); +} + +static int probe_prog_bind_map(int token_fd) +{ + char *cp, errmsg[STRERR_BUFSIZE]; + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + LIBBPF_OPTS(bpf_map_create_opts, map_opts, .token_fd = token_fd); + LIBBPF_OPTS(bpf_prog_load_opts, prog_opts, .token_fd = token_fd); + int ret, map, prog, insn_cnt = ARRAY_SIZE(insns); + + map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_det_bind", sizeof(int), 32, 1, &map_opts); + if (map < 0) { + ret = -errno; + cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); + pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n", + __func__, cp, -ret); + return ret; + } + + prog = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &prog_opts); + if (prog < 0) { + close(map); + return 0; + } + + ret = bpf_prog_bind_map(prog, map, NULL); + + close(map); + close(prog); + + return ret >= 0; +} + +static int probe_module_btf(int token_fd) +{ + static const char strs[] = "\0int"; + __u32 types[] = { + /* int */ + BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), + }; + struct bpf_btf_info info; + __u32 len = sizeof(info); + char name[16]; + int fd, err; + + fd = libbpf__load_raw_btf((char *)types, sizeof(types), strs, sizeof(strs), token_fd); + if (fd < 0) + return 0; /* BTF not supported at all */ + + memset(&info, 0, sizeof(info)); + info.name = ptr_to_u64(name); + info.name_len = sizeof(name); + + /* check that BPF_OBJ_GET_INFO_BY_FD supports specifying name pointer; + * kernel's module BTF support coincides with support for + * name/name_len fields in struct bpf_btf_info. + */ + err = bpf_btf_get_info_by_fd(fd, &info, &len); + close(fd); + return !err; +} + +static int probe_perf_link(int token_fd) +{ + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + LIBBPF_OPTS(bpf_prog_load_opts, opts, .token_fd = token_fd); + int prog_fd, link_fd, err; + + prog_fd = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", + insns, ARRAY_SIZE(insns), &opts); + if (prog_fd < 0) + return -errno; + + /* use invalid perf_event FD to get EBADF, if link is supported; + * otherwise EINVAL should be returned + */ + link_fd = bpf_link_create(prog_fd, -1, BPF_PERF_EVENT, NULL); + err = -errno; /* close() can clobber errno */ + + if (link_fd >= 0) + close(link_fd); + close(prog_fd); + + return link_fd < 0 && err == -EBADF; +} + +static int probe_uprobe_multi_link(int token_fd) +{ + LIBBPF_OPTS(bpf_prog_load_opts, load_opts, + .expected_attach_type = BPF_TRACE_UPROBE_MULTI, + .token_fd = token_fd, + ); + LIBBPF_OPTS(bpf_link_create_opts, link_opts); + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + int prog_fd, link_fd, err; + unsigned long offset = 0; + + prog_fd = bpf_prog_load(BPF_PROG_TYPE_KPROBE, NULL, "GPL", + insns, ARRAY_SIZE(insns), &load_opts); + if (prog_fd < 0) + return -errno; + + /* Creating uprobe in '/' binary should fail with -EBADF. */ + link_opts.uprobe_multi.path = "/"; + link_opts.uprobe_multi.offsets = &offset; + link_opts.uprobe_multi.cnt = 1; + + link_fd = bpf_link_create(prog_fd, -1, BPF_TRACE_UPROBE_MULTI, &link_opts); + err = -errno; /* close() can clobber errno */ + + if (link_fd >= 0) + close(link_fd); + close(prog_fd); + + return link_fd < 0 && err == -EBADF; +} + +static int probe_kern_bpf_cookie(int token_fd) +{ + struct bpf_insn insns[] = { + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_get_attach_cookie), + BPF_EXIT_INSN(), + }; + LIBBPF_OPTS(bpf_prog_load_opts, opts, .token_fd = token_fd); + int ret, insn_cnt = ARRAY_SIZE(insns); + + ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, &opts); + return probe_fd(ret); +} + +static int probe_kern_btf_enum64(int token_fd) +{ + static const char strs[] = "\0enum64"; + __u32 types[] = { + BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_ENUM64, 0, 0), 8), + }; + + return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), + strs, sizeof(strs), token_fd)); +} + +typedef int (*feature_probe_fn)(int /* token_fd */); + +static struct kern_feature_cache feature_cache; + +static struct kern_feature_desc { + const char *desc; + feature_probe_fn probe; +} feature_probes[__FEAT_CNT] = { + [FEAT_PROG_NAME] = { + "BPF program name", probe_kern_prog_name, + }, + [FEAT_GLOBAL_DATA] = { + "global variables", probe_kern_global_data, + }, + [FEAT_BTF] = { + "minimal BTF", probe_kern_btf, + }, + [FEAT_BTF_FUNC] = { + "BTF functions", probe_kern_btf_func, + }, + [FEAT_BTF_GLOBAL_FUNC] = { + "BTF global function", probe_kern_btf_func_global, + }, + [FEAT_BTF_DATASEC] = { + "BTF data section and variable", probe_kern_btf_datasec, + }, + [FEAT_ARRAY_MMAP] = { + "ARRAY map mmap()", probe_kern_array_mmap, + }, + [FEAT_EXP_ATTACH_TYPE] = { + "BPF_PROG_LOAD expected_attach_type attribute", + probe_kern_exp_attach_type, + }, + [FEAT_PROBE_READ_KERN] = { + "bpf_probe_read_kernel() helper", probe_kern_probe_read_kernel, + }, + [FEAT_PROG_BIND_MAP] = { + "BPF_PROG_BIND_MAP support", probe_prog_bind_map, + }, + [FEAT_MODULE_BTF] = { + "module BTF support", probe_module_btf, + }, + [FEAT_BTF_FLOAT] = { + "BTF_KIND_FLOAT support", probe_kern_btf_float, + }, + [FEAT_PERF_LINK] = { + "BPF perf link support", probe_perf_link, + }, + [FEAT_BTF_DECL_TAG] = { + "BTF_KIND_DECL_TAG support", probe_kern_btf_decl_tag, + }, + [FEAT_BTF_TYPE_TAG] = { + "BTF_KIND_TYPE_TAG support", probe_kern_btf_type_tag, + }, + [FEAT_MEMCG_ACCOUNT] = { + "memcg-based memory accounting", probe_memcg_account, + }, + [FEAT_BPF_COOKIE] = { + "BPF cookie support", probe_kern_bpf_cookie, + }, + [FEAT_BTF_ENUM64] = { + "BTF_KIND_ENUM64 support", probe_kern_btf_enum64, + }, + [FEAT_SYSCALL_WRAPPER] = { + "Kernel using syscall wrapper", probe_kern_syscall_wrapper, + }, + [FEAT_UPROBE_MULTI_LINK] = { + "BPF multi-uprobe link support", probe_uprobe_multi_link, + }, +}; + +bool feat_supported(struct kern_feature_cache *cache, enum kern_feature_id feat_id) +{ + struct kern_feature_desc *feat = &feature_probes[feat_id]; + int ret; + + /* assume global feature cache, unless custom one is provided */ + if (!cache) + cache = &feature_cache; + + if (READ_ONCE(cache->res[feat_id]) == FEAT_UNKNOWN) { + ret = feat->probe(cache->token_fd); + if (ret > 0) { + WRITE_ONCE(cache->res[feat_id], FEAT_SUPPORTED); + } else if (ret == 0) { + WRITE_ONCE(cache->res[feat_id], FEAT_MISSING); + } else { + pr_warn("Detection of kernel %s support failed: %d\n", feat->desc, ret); + WRITE_ONCE(cache->res[feat_id], FEAT_MISSING); + } + } + + return READ_ONCE(cache->res[feat_id]) == FEAT_SUPPORTED; +} diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index ea9b8158c20d69..4b5ff9508e18fd 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -59,6 +59,8 @@ #define BPF_FS_MAGIC 0xcafe4a11 #endif +#define BPF_FS_DEFAULT_PATH "/sys/fs/bpf" + #define BPF_INSN_SZ (sizeof(struct bpf_insn)) /* vsprintf() in __base_pr() uses nonliteral format string. It may break @@ -693,6 +695,10 @@ struct bpf_object { struct usdt_manager *usdt_man; + struct kern_feature_cache *feat_cache; + char *token_path; + int token_fd; + char path[]; }; @@ -2192,7 +2198,7 @@ static int build_map_pin_path(struct bpf_map *map, const char *path) int err; if (!path) - path = "/sys/fs/bpf"; + path = BPF_FS_DEFAULT_PATH; err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map)); if (err) @@ -3054,9 +3060,15 @@ static bool prog_needs_vmlinux_btf(struct bpf_program *prog) return false; } +static bool map_needs_vmlinux_btf(struct bpf_map *map) +{ + return bpf_map__is_struct_ops(map); +} + static bool obj_needs_vmlinux_btf(const struct bpf_object *obj) { struct bpf_program *prog; + struct bpf_map *map; int i; /* CO-RE relocations need kernel BTF, only when btf_custom_path @@ -3081,6 +3093,11 @@ static bool obj_needs_vmlinux_btf(const struct bpf_object *obj) return true; } + bpf_object__for_each_map(map, obj) { + if (map_needs_vmlinux_btf(map)) + return true; + } + return false; } @@ -3268,7 +3285,7 @@ static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj) } else { /* currently BPF_BTF_LOAD only supports log_level 1 */ err = btf_load_into_kernel(kern_btf, obj->log_buf, obj->log_size, - obj->log_level ? 1 : 0); + obj->log_level ? 1 : 0, obj->token_fd); } if (sanitize) { if (!err) { @@ -4591,6 +4608,63 @@ int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries) return 0; } +static int bpf_object_prepare_token(struct bpf_object *obj) +{ + const char *bpffs_path; + int bpffs_fd = -1, token_fd, err; + bool mandatory; + enum libbpf_print_level level; + + /* token is already set up */ + if (obj->token_fd > 0) + return 0; + /* token is explicitly prevented */ + if (obj->token_fd < 0) { + pr_debug("object '%s': token is prevented, skipping...\n", obj->name); + /* reset to zero to avoid extra checks during map_create and prog_load steps */ + obj->token_fd = 0; + return 0; + } + + mandatory = obj->token_path != NULL; + level = mandatory ? LIBBPF_WARN : LIBBPF_DEBUG; + + bpffs_path = obj->token_path ?: BPF_FS_DEFAULT_PATH; + bpffs_fd = open(bpffs_path, O_DIRECTORY, O_RDWR); + if (bpffs_fd < 0) { + err = -errno; + __pr(level, "object '%s': failed (%d) to open BPF FS mount at '%s'%s\n", + obj->name, err, bpffs_path, + mandatory ? "" : ", skipping optional step..."); + return mandatory ? err : 0; + } + + token_fd = bpf_token_create(bpffs_fd, 0); + close(bpffs_fd); + if (token_fd < 0) { + if (!mandatory && token_fd == -ENOENT) { + pr_debug("object '%s': BPF FS at '%s' doesn't have BPF token delegation set up, skipping...\n", + obj->name, bpffs_path); + return 0; + } + __pr(level, "object '%s': failed (%d) to create BPF token from '%s'%s\n", + obj->name, token_fd, bpffs_path, + mandatory ? "" : ", skipping optional step..."); + return mandatory ? token_fd : 0; + } + + obj->feat_cache = calloc(1, sizeof(*obj->feat_cache)); + if (!obj->feat_cache) { + close(token_fd); + return -ENOMEM; + } + + obj->token_fd = token_fd; + obj->feat_cache->token_fd = token_fd; + + return 0; +} + static int bpf_object__probe_loading(struct bpf_object *obj) { @@ -4600,6 +4674,7 @@ bpf_object__probe_loading(struct bpf_object *obj) BPF_EXIT_INSN(), }; int ret, insn_cnt = ARRAY_SIZE(insns); + LIBBPF_OPTS(bpf_prog_load_opts, opts, .token_fd = obj->token_fd); if (obj->gen_loader) return 0; @@ -4609,9 +4684,9 @@ bpf_object__probe_loading(struct bpf_object *obj) pr_warn("Failed to bump RLIMIT_MEMLOCK (err = %d), you might need to do it explicitly!\n", ret); /* make sure basic loading works */ - ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL); + ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &opts); if (ret < 0) - ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, NULL); + ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, &opts); if (ret < 0) { ret = errno; cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); @@ -4626,462 +4701,18 @@ bpf_object__probe_loading(struct bpf_object *obj) return 0; } -static int probe_fd(int fd) -{ - if (fd >= 0) - close(fd); - return fd >= 0; -} - -static int probe_kern_prog_name(void) -{ - const size_t attr_sz = offsetofend(union bpf_attr, prog_name); - struct bpf_insn insns[] = { - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }; - union bpf_attr attr; - int ret; - - memset(&attr, 0, attr_sz); - attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; - attr.license = ptr_to_u64("GPL"); - attr.insns = ptr_to_u64(insns); - attr.insn_cnt = (__u32)ARRAY_SIZE(insns); - libbpf_strlcpy(attr.prog_name, "libbpf_nametest", sizeof(attr.prog_name)); - - /* make sure loading with name works */ - ret = sys_bpf_prog_load(&attr, attr_sz, PROG_LOAD_ATTEMPTS); - return probe_fd(ret); -} - -static int probe_kern_global_data(void) -{ - char *cp, errmsg[STRERR_BUFSIZE]; - struct bpf_insn insns[] = { - BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16), - BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42), - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }; - int ret, map, insn_cnt = ARRAY_SIZE(insns); - - map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_global", sizeof(int), 32, 1, NULL); - if (map < 0) { - ret = -errno; - cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); - pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n", - __func__, cp, -ret); - return ret; - } - - insns[0].imm = map; - - ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL); - close(map); - return probe_fd(ret); -} - -static int probe_kern_btf(void) -{ - static const char strs[] = "\0int"; - __u32 types[] = { - /* int */ - BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), - }; - - return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), - strs, sizeof(strs))); -} - -static int probe_kern_btf_func(void) -{ - static const char strs[] = "\0int\0x\0a"; - /* void x(int a) {} */ - __u32 types[] = { - /* int */ - BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ - /* FUNC_PROTO */ /* [2] */ - BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), - BTF_PARAM_ENC(7, 1), - /* FUNC x */ /* [3] */ - BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2), - }; - - return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), - strs, sizeof(strs))); -} - -static int probe_kern_btf_func_global(void) -{ - static const char strs[] = "\0int\0x\0a"; - /* static void x(int a) {} */ - __u32 types[] = { - /* int */ - BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ - /* FUNC_PROTO */ /* [2] */ - BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), - BTF_PARAM_ENC(7, 1), - /* FUNC x BTF_FUNC_GLOBAL */ /* [3] */ - BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 2), - }; - - return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), - strs, sizeof(strs))); -} - -static int probe_kern_btf_datasec(void) -{ - static const char strs[] = "\0x\0.data"; - /* static int a; */ - __u32 types[] = { - /* int */ - BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ - /* VAR x */ /* [2] */ - BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1), - BTF_VAR_STATIC, - /* DATASEC val */ /* [3] */ - BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4), - BTF_VAR_SECINFO_ENC(2, 0, 4), - }; - - return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), - strs, sizeof(strs))); -} - -static int probe_kern_btf_float(void) -{ - static const char strs[] = "\0float"; - __u32 types[] = { - /* float */ - BTF_TYPE_FLOAT_ENC(1, 4), - }; - - return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), - strs, sizeof(strs))); -} - -static int probe_kern_btf_decl_tag(void) -{ - static const char strs[] = "\0tag"; - __u32 types[] = { - /* int */ - BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ - /* VAR x */ /* [2] */ - BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1), - BTF_VAR_STATIC, - /* attr */ - BTF_TYPE_DECL_TAG_ENC(1, 2, -1), - }; - - return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), - strs, sizeof(strs))); -} - -static int probe_kern_btf_type_tag(void) -{ - static const char strs[] = "\0tag"; - __u32 types[] = { - /* int */ - BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ - /* attr */ - BTF_TYPE_TYPE_TAG_ENC(1, 1), /* [2] */ - /* ptr */ - BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_PTR, 0, 0), 2), /* [3] */ - }; - - return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), - strs, sizeof(strs))); -} - -static int probe_kern_array_mmap(void) -{ - LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = BPF_F_MMAPABLE); - int fd; - - fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_mmap", sizeof(int), sizeof(int), 1, &opts); - return probe_fd(fd); -} - -static int probe_kern_exp_attach_type(void) -{ - LIBBPF_OPTS(bpf_prog_load_opts, opts, .expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE); - struct bpf_insn insns[] = { - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }; - int fd, insn_cnt = ARRAY_SIZE(insns); - - /* use any valid combination of program type and (optional) - * non-zero expected attach type (i.e., not a BPF_CGROUP_INET_INGRESS) - * to see if kernel supports expected_attach_type field for - * BPF_PROG_LOAD command - */ - fd = bpf_prog_load(BPF_PROG_TYPE_CGROUP_SOCK, NULL, "GPL", insns, insn_cnt, &opts); - return probe_fd(fd); -} - -static int probe_kern_probe_read_kernel(void) -{ - struct bpf_insn insns[] = { - BPF_MOV64_REG(BPF_REG_1, BPF_REG_10), /* r1 = r10 (fp) */ - BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8), /* r1 += -8 */ - BPF_MOV64_IMM(BPF_REG_2, 8), /* r2 = 8 */ - BPF_MOV64_IMM(BPF_REG_3, 0), /* r3 = 0 */ - BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_probe_read_kernel), - BPF_EXIT_INSN(), - }; - int fd, insn_cnt = ARRAY_SIZE(insns); - - fd = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, NULL); - return probe_fd(fd); -} - -static int probe_prog_bind_map(void) -{ - char *cp, errmsg[STRERR_BUFSIZE]; - struct bpf_insn insns[] = { - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }; - int ret, map, prog, insn_cnt = ARRAY_SIZE(insns); - - map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_det_bind", sizeof(int), 32, 1, NULL); - if (map < 0) { - ret = -errno; - cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); - pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n", - __func__, cp, -ret); - return ret; - } - - prog = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL); - if (prog < 0) { - close(map); - return 0; - } - - ret = bpf_prog_bind_map(prog, map, NULL); - - close(map); - close(prog); - - return ret >= 0; -} - -static int probe_module_btf(void) -{ - static const char strs[] = "\0int"; - __u32 types[] = { - /* int */ - BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), - }; - struct bpf_btf_info info; - __u32 len = sizeof(info); - char name[16]; - int fd, err; - - fd = libbpf__load_raw_btf((char *)types, sizeof(types), strs, sizeof(strs)); - if (fd < 0) - return 0; /* BTF not supported at all */ - - memset(&info, 0, sizeof(info)); - info.name = ptr_to_u64(name); - info.name_len = sizeof(name); - - /* check that BPF_OBJ_GET_INFO_BY_FD supports specifying name pointer; - * kernel's module BTF support coincides with support for - * name/name_len fields in struct bpf_btf_info. - */ - err = bpf_btf_get_info_by_fd(fd, &info, &len); - close(fd); - return !err; -} - -static int probe_perf_link(void) -{ - struct bpf_insn insns[] = { - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }; - int prog_fd, link_fd, err; - - prog_fd = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", - insns, ARRAY_SIZE(insns), NULL); - if (prog_fd < 0) - return -errno; - - /* use invalid perf_event FD to get EBADF, if link is supported; - * otherwise EINVAL should be returned - */ - link_fd = bpf_link_create(prog_fd, -1, BPF_PERF_EVENT, NULL); - err = -errno; /* close() can clobber errno */ - - if (link_fd >= 0) - close(link_fd); - close(prog_fd); - - return link_fd < 0 && err == -EBADF; -} - -static int probe_uprobe_multi_link(void) -{ - LIBBPF_OPTS(bpf_prog_load_opts, load_opts, - .expected_attach_type = BPF_TRACE_UPROBE_MULTI, - ); - LIBBPF_OPTS(bpf_link_create_opts, link_opts); - struct bpf_insn insns[] = { - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }; - int prog_fd, link_fd, err; - unsigned long offset = 0; - - prog_fd = bpf_prog_load(BPF_PROG_TYPE_KPROBE, NULL, "GPL", - insns, ARRAY_SIZE(insns), &load_opts); - if (prog_fd < 0) - return -errno; - - /* Creating uprobe in '/' binary should fail with -EBADF. */ - link_opts.uprobe_multi.path = "/"; - link_opts.uprobe_multi.offsets = &offset; - link_opts.uprobe_multi.cnt = 1; - - link_fd = bpf_link_create(prog_fd, -1, BPF_TRACE_UPROBE_MULTI, &link_opts); - err = -errno; /* close() can clobber errno */ - - if (link_fd >= 0) - close(link_fd); - close(prog_fd); - - return link_fd < 0 && err == -EBADF; -} - -static int probe_kern_bpf_cookie(void) -{ - struct bpf_insn insns[] = { - BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_get_attach_cookie), - BPF_EXIT_INSN(), - }; - int ret, insn_cnt = ARRAY_SIZE(insns); - - ret = bpf_prog_load(BPF_PROG_TYPE_KPROBE, NULL, "GPL", insns, insn_cnt, NULL); - return probe_fd(ret); -} - -static int probe_kern_btf_enum64(void) -{ - static const char strs[] = "\0enum64"; - __u32 types[] = { - BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_ENUM64, 0, 0), 8), - }; - - return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), - strs, sizeof(strs))); -} - -static int probe_kern_syscall_wrapper(void); - -enum kern_feature_result { - FEAT_UNKNOWN = 0, - FEAT_SUPPORTED = 1, - FEAT_MISSING = 2, -}; - -typedef int (*feature_probe_fn)(void); - -static struct kern_feature_desc { - const char *desc; - feature_probe_fn probe; - enum kern_feature_result res; -} feature_probes[__FEAT_CNT] = { - [FEAT_PROG_NAME] = { - "BPF program name", probe_kern_prog_name, - }, - [FEAT_GLOBAL_DATA] = { - "global variables", probe_kern_global_data, - }, - [FEAT_BTF] = { - "minimal BTF", probe_kern_btf, - }, - [FEAT_BTF_FUNC] = { - "BTF functions", probe_kern_btf_func, - }, - [FEAT_BTF_GLOBAL_FUNC] = { - "BTF global function", probe_kern_btf_func_global, - }, - [FEAT_BTF_DATASEC] = { - "BTF data section and variable", probe_kern_btf_datasec, - }, - [FEAT_ARRAY_MMAP] = { - "ARRAY map mmap()", probe_kern_array_mmap, - }, - [FEAT_EXP_ATTACH_TYPE] = { - "BPF_PROG_LOAD expected_attach_type attribute", - probe_kern_exp_attach_type, - }, - [FEAT_PROBE_READ_KERN] = { - "bpf_probe_read_kernel() helper", probe_kern_probe_read_kernel, - }, - [FEAT_PROG_BIND_MAP] = { - "BPF_PROG_BIND_MAP support", probe_prog_bind_map, - }, - [FEAT_MODULE_BTF] = { - "module BTF support", probe_module_btf, - }, - [FEAT_BTF_FLOAT] = { - "BTF_KIND_FLOAT support", probe_kern_btf_float, - }, - [FEAT_PERF_LINK] = { - "BPF perf link support", probe_perf_link, - }, - [FEAT_BTF_DECL_TAG] = { - "BTF_KIND_DECL_TAG support", probe_kern_btf_decl_tag, - }, - [FEAT_BTF_TYPE_TAG] = { - "BTF_KIND_TYPE_TAG support", probe_kern_btf_type_tag, - }, - [FEAT_MEMCG_ACCOUNT] = { - "memcg-based memory accounting", probe_memcg_account, - }, - [FEAT_BPF_COOKIE] = { - "BPF cookie support", probe_kern_bpf_cookie, - }, - [FEAT_BTF_ENUM64] = { - "BTF_KIND_ENUM64 support", probe_kern_btf_enum64, - }, - [FEAT_SYSCALL_WRAPPER] = { - "Kernel using syscall wrapper", probe_kern_syscall_wrapper, - }, - [FEAT_UPROBE_MULTI_LINK] = { - "BPF multi-uprobe link support", probe_uprobe_multi_link, - }, -}; - bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id) { - struct kern_feature_desc *feat = &feature_probes[feat_id]; - int ret; - if (obj && obj->gen_loader) /* To generate loader program assume the latest kernel * to avoid doing extra prog_load, map_create syscalls. */ return true; - if (READ_ONCE(feat->res) == FEAT_UNKNOWN) { - ret = feat->probe(); - if (ret > 0) { - WRITE_ONCE(feat->res, FEAT_SUPPORTED); - } else if (ret == 0) { - WRITE_ONCE(feat->res, FEAT_MISSING); - } else { - pr_warn("Detection of kernel %s support failed: %d\n", feat->desc, ret); - WRITE_ONCE(feat->res, FEAT_MISSING); - } - } + if (obj->token_fd) + return feat_supported(obj->feat_cache, feat_id); - return READ_ONCE(feat->res) == FEAT_SUPPORTED; + return feat_supported(NULL, feat_id); } static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd) @@ -5200,6 +4831,7 @@ static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, b create_attr.map_flags = def->map_flags; create_attr.numa_node = map->numa_node; create_attr.map_extra = map->map_extra; + create_attr.token_fd = obj->token_fd; if (bpf_map__is_struct_ops(map)) create_attr.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id; @@ -7035,6 +6667,7 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog load_attr.attach_btf_id = prog->attach_btf_id; load_attr.kern_version = kern_version; load_attr.prog_ifindex = prog->prog_ifindex; + load_attr.token_fd = obj->token_fd; /* specify func_info/line_info only if kernel supports them */ btf_fd = bpf_object__btf_fd(obj); @@ -7496,10 +7129,10 @@ static int bpf_object_init_progs(struct bpf_object *obj, const struct bpf_object static struct bpf_object *bpf_object_open(const char *path, const void *obj_buf, size_t obj_buf_sz, const struct bpf_object_open_opts *opts) { - const char *obj_name, *kconfig, *btf_tmp_path; + const char *obj_name, *kconfig, *btf_tmp_path, *token_path; struct bpf_object *obj; char tmp_name[64]; - int err; + int err, token_fd; char *log_buf; size_t log_size; __u32 log_level; @@ -7533,6 +7166,28 @@ static struct bpf_object *bpf_object_open(const char *path, const void *obj_buf, if (log_size && !log_buf) return ERR_PTR(-EINVAL); + token_path = OPTS_GET(opts, bpf_token_path, NULL); + token_fd = OPTS_GET(opts, bpf_token_fd, -1); + /* non-empty token path can't be combined with invalid token FD */ + if (token_path && token_path[0] != '\0' && token_fd < 0) + return ERR_PTR(-EINVAL); + /* empty token path can't be combined with valid token FD */ + if (token_path && token_path[0] == '\0' && token_fd > 0) + return ERR_PTR(-EINVAL); + /* if user didn't specify bpf_token_path/bpf_token_fd explicitly, + * check if LIBBPF_BPF_TOKEN_PATH envvar was set and treat it as + * bpf_token_path option + */ + if (token_fd == 0 && !token_path) + token_path = getenv("LIBBPF_BPF_TOKEN_PATH"); + /* empty token_path is equivalent to invalid token_fd */ + if (token_path && token_path[0] == '\0') { + token_path = NULL; + token_fd = -1; + } + if (token_path && strlen(token_path) >= PATH_MAX) + return ERR_PTR(-ENAMETOOLONG); + obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name); if (IS_ERR(obj)) return obj; @@ -7541,6 +7196,19 @@ static struct bpf_object *bpf_object_open(const char *path, const void *obj_buf, obj->log_size = log_size; obj->log_level = log_level; + obj->token_fd = token_fd <= 0 ? token_fd : dup_good_fd(token_fd); + if (token_fd > 0 && obj->token_fd < 0) { + err = -errno; + goto out; + } + if (token_path) { + obj->token_path = strdup(token_path); + if (!obj->token_path) { + err = -ENOMEM; + goto out; + } + } + btf_tmp_path = OPTS_GET(opts, btf_custom_path, NULL); if (btf_tmp_path) { if (strlen(btf_tmp_path) >= PATH_MAX) { @@ -8051,7 +7719,8 @@ static int bpf_object_load(struct bpf_object *obj, int extra_log_level, const ch if (obj->gen_loader) bpf_gen__init(obj->gen_loader, extra_log_level, obj->nr_programs, obj->nr_maps); - err = bpf_object__probe_loading(obj); + err = bpf_object_prepare_token(obj); + err = err ? : bpf_object__probe_loading(obj); err = err ? : bpf_object__load_vmlinux_btf(obj, false); err = err ? : bpf_object__resolve_externs(obj, obj->kconfig); err = err ? : bpf_object__sanitize_and_load_btf(obj); @@ -8588,6 +8257,11 @@ void bpf_object__close(struct bpf_object *obj) } zfree(&obj->programs); + zfree(&obj->feat_cache); + zfree(&obj->token_path); + if (obj->token_fd > 0) + close(obj->token_fd); + free(obj); } @@ -10601,7 +10275,7 @@ static const char *arch_specific_syscall_pfx(void) #endif } -static int probe_kern_syscall_wrapper(void) +int probe_kern_syscall_wrapper(int token_fd) { char syscall_name[64]; const char *ksys_pfx; diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h index 6cd9c501624f50..916904bd2a7ad1 100644 --- a/tools/lib/bpf/libbpf.h +++ b/tools/lib/bpf/libbpf.h @@ -177,10 +177,45 @@ struct bpf_object_open_opts { * logs through its print callback. */ __u32 kernel_log_level; + /* FD of a BPF token instantiated by user through bpf_token_create() + * API. BPF object will keep dup()'ed FD internally, so passed token + * FD can be closed after BPF object/skeleton open step. + * + * Setting bpf_token_fd to negative value disables libbpf's automatic + * attempt to create BPF token from default BPF FS mount point + * (/sys/fs/bpf), in case this default behavior is undesirable. + * + * If bpf_token_path and bpf_token_fd are not specified, libbpf will + * consult LIBBPF_BPF_TOKEN_PATH environment variable. If set, it will + * be taken as a value of bpf_token_path option and will force libbpf + * to either create BPF token from provided custom BPF FS path, or + * will disable implicit BPF token creation, if envvar value is an + * empty string. + * + * bpf_token_path and bpf_token_fd are mutually exclusive and only one + * of those options should be set. Either of them overrides + * LIBBPF_BPF_TOKEN_PATH envvar. + */ + int bpf_token_fd; + /* Path to BPF FS mount point to derive BPF token from. + * + * Created BPF token will be used for all bpf() syscall operations + * that accept BPF token (e.g., map creation, BTF and program loads, + * etc) automatically within instantiated BPF object. + * + * Setting bpf_token_path option to empty string disables libbpf's + * automatic attempt to create BPF token from default BPF FS mount + * point (/sys/fs/bpf), in case this default behavior is undesirable. + * + * bpf_token_path and bpf_token_fd are mutually exclusive and only one + * of those options should be set. Either of them overrides + * LIBBPF_BPF_TOKEN_PATH envvar. + */ + const char *bpf_token_path; size_t :0; }; -#define bpf_object_open_opts__last_field kernel_log_level +#define bpf_object_open_opts__last_field bpf_token_path /** * @brief **bpf_object__open()** creates a bpf_object by opening diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map index 91c5aef7dae7de..df7657b65c47b2 100644 --- a/tools/lib/bpf/libbpf.map +++ b/tools/lib/bpf/libbpf.map @@ -401,6 +401,7 @@ LIBBPF_1.3.0 { bpf_program__attach_netkit; bpf_program__attach_tcx; bpf_program__attach_uprobe_multi; + bpf_token_create; ring__avail_data_size; ring__consume; ring__consumer_pos; diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h index b5d334754e5dc8..4cda32298c49c4 100644 --- a/tools/lib/bpf/libbpf_internal.h +++ b/tools/lib/bpf/libbpf_internal.h @@ -360,15 +360,32 @@ enum kern_feature_id { __FEAT_CNT, }; -int probe_memcg_account(void); +enum kern_feature_result { + FEAT_UNKNOWN = 0, + FEAT_SUPPORTED = 1, + FEAT_MISSING = 2, +}; + +struct kern_feature_cache { + enum kern_feature_result res[__FEAT_CNT]; + int token_fd; +}; + +bool feat_supported(struct kern_feature_cache *cache, enum kern_feature_id feat_id); bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id); + +int probe_kern_syscall_wrapper(int token_fd); +int probe_memcg_account(int token_fd); int bump_rlimit_memlock(void); int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz); int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz); int libbpf__load_raw_btf(const char *raw_types, size_t types_len, - const char *str_sec, size_t str_len); -int btf_load_into_kernel(struct btf *btf, char *log_buf, size_t log_sz, __u32 log_level); + const char *str_sec, size_t str_len, + int token_fd); +int btf_load_into_kernel(struct btf *btf, + char *log_buf, size_t log_sz, __u32 log_level, + int token_fd); struct btf *btf_get_from_fd(int btf_fd, struct btf *base_btf); void btf_get_kernel_prefix_kind(enum bpf_attach_type attach_type, @@ -532,6 +549,17 @@ static inline bool is_ldimm64_insn(struct bpf_insn *insn) return insn->code == (BPF_LD | BPF_IMM | BPF_DW); } +/* Unconditionally dup FD, ensuring it doesn't use [0, 2] range. + * Original FD is not closed or altered in any other way. + * Preserves original FD value, if it's invalid (negative). + */ +static inline int dup_good_fd(int fd) +{ + if (fd < 0) + return fd; + return fcntl(fd, F_DUPFD_CLOEXEC, 3); +} + /* if fd is stdin, stdout, or stderr, dup to a fd greater than 2 * Takes ownership of the fd passed in, and closes it if calling * fcntl(fd, F_DUPFD_CLOEXEC, 3). @@ -543,7 +571,7 @@ static inline int ensure_good_fd(int fd) if (fd < 0) return fd; if (fd < 3) { - fd = fcntl(fd, F_DUPFD_CLOEXEC, 3); + fd = dup_good_fd(fd); saved_errno = errno; close(old_fd); errno = saved_errno; diff --git a/tools/lib/bpf/libbpf_probes.c b/tools/lib/bpf/libbpf_probes.c index 9c4db90b92b6b5..8e743700663916 100644 --- a/tools/lib/bpf/libbpf_probes.c +++ b/tools/lib/bpf/libbpf_probes.c @@ -219,7 +219,8 @@ int libbpf_probe_bpf_prog_type(enum bpf_prog_type prog_type, const void *opts) } int libbpf__load_raw_btf(const char *raw_types, size_t types_len, - const char *str_sec, size_t str_len) + const char *str_sec, size_t str_len, + int token_fd) { struct btf_header hdr = { .magic = BTF_MAGIC, @@ -229,6 +230,7 @@ int libbpf__load_raw_btf(const char *raw_types, size_t types_len, .str_off = types_len, .str_len = str_len, }; + LIBBPF_OPTS(bpf_btf_load_opts, opts, .token_fd = token_fd); int btf_fd, btf_len; __u8 *raw_btf; @@ -241,7 +243,7 @@ int libbpf__load_raw_btf(const char *raw_types, size_t types_len, memcpy(raw_btf + hdr.hdr_len, raw_types, hdr.type_len); memcpy(raw_btf + hdr.hdr_len + hdr.type_len, str_sec, hdr.str_len); - btf_fd = bpf_btf_load(raw_btf, btf_len, NULL); + btf_fd = bpf_btf_load(raw_btf, btf_len, &opts); free(raw_btf); return btf_fd; @@ -271,7 +273,7 @@ static int load_local_storage_btf(void) }; return libbpf__load_raw_btf((char *)types, sizeof(types), - strs, sizeof(strs)); + strs, sizeof(strs), 0); } static int probe_map_create(enum bpf_map_type map_type) diff --git a/tools/lib/bpf/linker.c b/tools/lib/bpf/linker.c index 5ced96d99f8c5f..52a2901e8bd0ec 100644 --- a/tools/lib/bpf/linker.c +++ b/tools/lib/bpf/linker.c @@ -719,13 +719,25 @@ static int linker_sanity_check_elf(struct src_obj *obj) return -EINVAL; } - if (sec->shdr->sh_addralign && !is_pow_of_2(sec->shdr->sh_addralign)) + if (sec->shdr->sh_addralign && !is_pow_of_2(sec->shdr->sh_addralign)) { + pr_warn("ELF section #%zu alignment %llu is non pow-of-2 alignment in %s\n", + sec->sec_idx, (long long unsigned)sec->shdr->sh_addralign, + obj->filename); return -EINVAL; - if (sec->shdr->sh_addralign != sec->data->d_align) + } + if (sec->shdr->sh_addralign != sec->data->d_align) { + pr_warn("ELF section #%zu has inconsistent alignment addr=%llu != d=%llu in %s\n", + sec->sec_idx, (long long unsigned)sec->shdr->sh_addralign, + (long long unsigned)sec->data->d_align, obj->filename); return -EINVAL; + } - if (sec->shdr->sh_size != sec->data->d_size) + if (sec->shdr->sh_size != sec->data->d_size) { + pr_warn("ELF section #%zu has inconsistent section size sh=%llu != d=%llu in %s\n", + sec->sec_idx, (long long unsigned)sec->shdr->sh_size, + (long long unsigned)sec->data->d_size, obj->filename); return -EINVAL; + } switch (sec->shdr->sh_type) { case SHT_SYMTAB: @@ -737,8 +749,12 @@ static int linker_sanity_check_elf(struct src_obj *obj) break; case SHT_PROGBITS: if (sec->shdr->sh_flags & SHF_EXECINSTR) { - if (sec->shdr->sh_size % sizeof(struct bpf_insn) != 0) + if (sec->shdr->sh_size % sizeof(struct bpf_insn) != 0) { + pr_warn("ELF section #%zu has unexpected size alignment %llu in %s\n", + sec->sec_idx, (long long unsigned)sec->shdr->sh_size, + obj->filename); return -EINVAL; + } } break; case SHT_NOBITS: diff --git a/tools/lib/bpf/str_error.h b/tools/lib/bpf/str_error.h index a139334d57b603..626d7ffb03d60d 100644 --- a/tools/lib/bpf/str_error.h +++ b/tools/lib/bpf/str_error.h @@ -2,5 +2,8 @@ #ifndef __LIBBPF_STR_ERROR_H #define __LIBBPF_STR_ERROR_H +#define STRERR_BUFSIZE 128 + char *libbpf_strerror_r(int err, char *dst, int len); + #endif /* __LIBBPF_STR_ERROR_H */ diff --git a/tools/net/ynl/lib/nlspec.py b/tools/net/ynl/lib/nlspec.py index 92889298b197e5..44f13e383e8a94 100644 --- a/tools/net/ynl/lib/nlspec.py +++ b/tools/net/ynl/lib/nlspec.py @@ -158,6 +158,9 @@ class SpecAttr(SpecElement): len integer, optional byte length of binary types display_hint string, hint to help choose format specifier when displaying the value + sub_message string, name of sub message type + selector string, name of attribute used to select + sub-message type is_auto_scalar bool, attr is a variable-size scalar """ @@ -173,6 +176,8 @@ def __init__(self, family, attr_set, yaml, value): self.byte_order = yaml.get('byte-order') self.len = yaml.get('len') self.display_hint = yaml.get('display-hint') + self.sub_message = yaml.get('sub-message') + self.selector = yaml.get('selector') self.is_auto_scalar = self.type == "sint" or self.type == "uint" @@ -278,6 +283,47 @@ def items(self): return self.members.items() +class SpecSubMessage(SpecElement): + """ Netlink sub-message definition + + Represents a set of sub-message formats for polymorphic nlattrs + that contain type-specific sub messages. + + Attributes: + name string, name of sub-message definition + formats dict of sub-message formats indexed by match value + """ + def __init__(self, family, yaml): + super().__init__(family, yaml) + + self.formats = collections.OrderedDict() + for elem in self.yaml['formats']: + format = self.new_format(family, elem) + self.formats[format.value] = format + + def new_format(self, family, format): + return SpecSubMessageFormat(family, format) + + +class SpecSubMessageFormat(SpecElement): + """ Netlink sub-message definition + + Represents a set of sub-message formats for polymorphic nlattrs + that contain type-specific sub messages. + + Attributes: + value attribute value to match against type selector + fixed_header string, name of fixed header, or None + attr_set string, name of attribute set, or None + """ + def __init__(self, family, yaml): + super().__init__(family, yaml) + + self.value = yaml.get('value') + self.fixed_header = yaml.get('fixed-header') + self.attr_set = yaml.get('attribute-set') + + class SpecOperation(SpecElement): """Netlink Operation @@ -365,6 +411,7 @@ class SpecFamily(SpecElement): attr_sets dict of attribute sets msgs dict of all messages (index by name) + sub_msgs dict of all sub messages (index by name) ops dict of all valid requests / responses ntfs dict of all async events consts dict of all constants/enums @@ -405,6 +452,7 @@ def __init__(self, spec_path, schema_path=None, exclude_ops=None): jsonschema.validate(self.yaml, schema) self.attr_sets = collections.OrderedDict() + self.sub_msgs = collections.OrderedDict() self.msgs = collections.OrderedDict() self.req_by_value = collections.OrderedDict() self.rsp_by_value = collections.OrderedDict() @@ -441,6 +489,9 @@ def new_attr_set(self, elem): def new_struct(self, elem): return SpecStruct(self, elem) + def new_sub_message(self, elem): + return SpecSubMessage(self, elem); + def new_operation(self, elem, req_val, rsp_val): return SpecOperation(self, elem, req_val, rsp_val) @@ -529,6 +580,10 @@ def resolve(self): attr_set = self.new_attr_set(elem) self.attr_sets[elem['name']] = attr_set + for elem in self.yaml.get('sub-messages', []): + sub_message = self.new_sub_message(elem) + self.sub_msgs[sub_message.name] = sub_message + if self.msg_id_model == 'unified': self._dictify_ops_unified() elif self.msg_id_model == 'directional': diff --git a/tools/net/ynl/lib/ynl.py b/tools/net/ynl/lib/ynl.py index c56dad9593c62e..1e10512b2117cb 100644 --- a/tools/net/ynl/lib/ynl.py +++ b/tools/net/ynl/lib/ynl.py @@ -98,12 +98,12 @@ class NlAttr: } def __init__(self, raw, offset): - self._len, self._type = struct.unpack("HH", raw[offset:offset + 4]) + self._len, self._type = struct.unpack("HH", raw[offset : offset + 4]) self.type = self._type & ~Netlink.NLA_TYPE_MASK self.is_nest = self._type & Netlink.NLA_F_NESTED self.payload_len = self._len self.full_len = (self.payload_len + 3) & ~3 - self.raw = raw[offset + 4:offset + self.payload_len] + self.raw = raw[offset + 4 : offset + self.payload_len] @classmethod def get_format(cls, attr_type, byte_order=None): @@ -154,7 +154,7 @@ def as_struct(self, members): for m in members: # TODO: handle non-scalar members if m.type == 'binary': - decoded = self.raw[offset:offset+m['len']] + decoded = self.raw[offset : offset + m['len']] offset += m['len'] elif m.type in NlAttr.type_formats: format = self.get_format(m.type, m.byte_order) @@ -170,10 +170,9 @@ def __repr__(self): class NlAttrs: - def __init__(self, msg): + def __init__(self, msg, offset=0): self.attrs = [] - offset = 0 while offset < len(msg): attr = NlAttr(msg, offset) offset += attr.full_len @@ -193,12 +192,12 @@ def __repr__(self): class NlMsg: def __init__(self, msg, offset, attr_space=None): - self.hdr = msg[offset:offset + 16] + self.hdr = msg[offset : offset + 16] self.nl_len, self.nl_type, self.nl_flags, self.nl_seq, self.nl_portid = \ struct.unpack("IHHII", self.hdr) - self.raw = msg[offset + 16:offset + self.nl_len] + self.raw = msg[offset + 16 : offset + self.nl_len] self.error = 0 self.done = 0 @@ -371,8 +370,8 @@ def decode(self, ynl, nl_msg): fixed_header_size = 0 if ynl: op = ynl.rsp_by_value[msg.cmd()] - fixed_header_size = ynl._fixed_header_size(op) - msg.raw_attrs = NlAttrs(msg.raw[fixed_header_size:]) + fixed_header_size = ynl._fixed_header_size(op.fixed_header) + msg.raw_attrs = NlAttrs(msg.raw, fixed_header_size) return msg def get_mcast_id(self, mcast_name, mcast_groups): @@ -549,6 +548,37 @@ def _rsp_add(self, rsp, name, is_multi, decoded): else: rsp[name] = [decoded] + def _resolve_selector(self, attr_spec, vals): + sub_msg = attr_spec.sub_message + if sub_msg not in self.sub_msgs: + raise Exception(f"No sub-message spec named {sub_msg} for {attr_spec.name}") + sub_msg_spec = self.sub_msgs[sub_msg] + + selector = attr_spec.selector + if selector not in vals: + raise Exception(f"There is no value for {selector} to resolve '{attr_spec.name}'") + value = vals[selector] + if value not in sub_msg_spec.formats: + raise Exception(f"No message format for '{value}' in sub-message spec '{sub_msg}'") + + spec = sub_msg_spec.formats[value] + return spec + + def _decode_sub_msg(self, attr, attr_spec, rsp): + msg_format = self._resolve_selector(attr_spec, rsp) + decoded = {} + offset = 0 + if msg_format.fixed_header: + decoded.update(self._decode_fixed_header(attr, msg_format.fixed_header)); + offset = self._fixed_header_size(msg_format.fixed_header) + if msg_format.attr_set: + if msg_format.attr_set in self.attr_sets: + subdict = self._decode(NlAttrs(attr.raw, offset), msg_format.attr_set) + decoded.update(subdict) + else: + raise Exception(f"Unknown attribute-set '{attr_space}' when decoding '{attr_spec.name}'") + return decoded + def _decode(self, attrs, space): if space: attr_space = self.attr_sets[space] @@ -586,6 +616,8 @@ def _decode(self, attrs, space): value = self._decode_enum(value, attr_spec) selector = self._decode_enum(selector, attr_spec) decoded = {"value": value, "selector": selector} + elif attr_spec["type"] == 'sub-message': + decoded = self._decode_sub_msg(attr, attr_spec, rsp) else: if not self.process_unknown: raise Exception(f'Unknown {attr_spec["type"]} with name {attr_spec["name"]}') @@ -626,20 +658,23 @@ def _decode_extack(self, request, op, extack): return msg = self.nlproto.decode(self, NlMsg(request, 0, op.attr_set)) - offset = 20 + self._fixed_header_size(op) + offset = 20 + self._fixed_header_size(op.fixed_header) path = self._decode_extack_path(msg.raw_attrs, op.attr_set, offset, extack['bad-attr-offs']) if path: del extack['bad-attr-offs'] extack['bad-attr'] = path - def _fixed_header_size(self, op): - if op.fixed_header: - fixed_header_members = self.consts[op.fixed_header].members + def _fixed_header_size(self, name): + if name: + fixed_header_members = self.consts[name].members size = 0 for m in fixed_header_members: - format = NlAttr.get_format(m.type, m.byte_order) - size += format.size + if m.type in ['pad', 'binary']: + size += m.len + else: + format = NlAttr.get_format(m.type, m.byte_order) + size += format.size return size else: return 0 @@ -649,12 +684,20 @@ def _decode_fixed_header(self, msg, name): fixed_header_attrs = dict() offset = 0 for m in fixed_header_members: - format = NlAttr.get_format(m.type, m.byte_order) - [ value ] = format.unpack_from(msg.raw, offset) - offset += format.size - if m.enum: - value = self._decode_enum(value, m) - fixed_header_attrs[m.name] = value + value = None + if m.type == 'pad': + offset += m.len + elif m.type == 'binary': + value = msg.raw[offset : offset + m.len] + offset += m.len + else: + format = NlAttr.get_format(m.type, m.byte_order) + [ value ] = format.unpack_from(msg.raw, offset) + offset += format.size + if value is not None: + if m.enum: + value = self._decode_enum(value, m) + fixed_header_attrs[m.name] = value return fixed_header_attrs def handle_ntf(self, decoded): @@ -721,8 +764,13 @@ def _op(self, method, vals, flags=None, dump=False): fixed_header_members = self.consts[op.fixed_header].members for m in fixed_header_members: value = vals.pop(m.name) if m.name in vals else 0 - format = NlAttr.get_format(m.type, m.byte_order) - msg += format.pack(value) + if m.type == 'pad': + msg += bytearray(m.len) + elif m.type == 'binary': + msg += bytes.fromhex(value) + else: + format = NlAttr.get_format(m.type, m.byte_order) + msg += format.pack(value) for name, value in vals.items(): msg += self._add_attr(op.attr_set.name, name, value) msg = _genl_msg_finalize(msg) diff --git a/tools/net/ynl/ynl-gen-rst.py b/tools/net/ynl/ynl-gen-rst.py index 8c62e040df5dce..262d88f886969d 100755 --- a/tools/net/ynl/ynl-gen-rst.py +++ b/tools/net/ynl/ynl-gen-rst.py @@ -69,7 +69,7 @@ def rst_paragraph(paragraph: str, level: int = 0) -> str: def rst_bullet(item: str, level: int = 0) -> str: """Return a formatted a bullet""" - return headroom(level) + f" - {item}" + return headroom(level) + f"- {item}" def rst_subsection(title: str) -> str: @@ -240,7 +240,7 @@ def parse_attr_sets(entries: List[Dict[str, Any]]) -> str: lines.append(rst_section(entry["name"])) for attr in entry["attributes"]: type_ = attr.get("type") - attr_line = bold(attr["name"]) + attr_line = attr["name"] if type_: # Add the attribute type in the same line attr_line += f" ({inline(type_)})" @@ -250,7 +250,25 @@ def parse_attr_sets(entries: List[Dict[str, Any]]) -> str: for k in attr.keys(): if k in preprocessed + ignored: continue - lines.append(rst_fields(k, sanitize(attr[k]), 2)) + lines.append(rst_fields(k, sanitize(attr[k]), 0)) + lines.append("\n") + + return "\n".join(lines) + + +def parse_sub_messages(entries: List[Dict[str, Any]]) -> str: + """Parse sub-message definitions""" + lines = [] + + for entry in entries: + lines.append(rst_section(entry["name"])) + for fmt in entry["formats"]: + value = fmt["value"] + + lines.append(rst_bullet(bold(value))) + for attr in ['fixed-header', 'attribute-set']: + if attr in fmt: + lines.append(rst_fields(attr, fmt[attr], 1)) lines.append("\n") return "\n".join(lines) @@ -292,6 +310,11 @@ def parse_yaml(obj: Dict[str, Any]) -> str: lines.append(rst_subtitle("Attribute sets")) lines.append(parse_attr_sets(obj["attribute-sets"])) + # Sub-messages + if "sub-messages" in obj: + lines.append(rst_subtitle("Sub-messages")) + lines.append(parse_sub_messages(obj["sub-messages"])) + return "\n".join(lines) @@ -360,7 +383,7 @@ def generate_main_index_rst(output: str) -> None: index_dir = os.path.dirname(output) logging.debug("Looking for .rst files in %s", index_dir) - for filename in os.listdir(index_dir): + for filename in sorted(os.listdir(index_dir)): if not filename.endswith(".rst") or filename == "index.rst": continue lines.append(f" {filename.replace('.rst', '')}\n") diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index 3b2061d1c1a527..f0c854d6511c63 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -58,6 +58,7 @@ TARGETS += net/forwarding TARGETS += net/hsr TARGETS += net/mptcp TARGETS += net/openvswitch +TARGETS += net/tcp_ao TARGETS += netfilter TARGETS += nsfs TARGETS += perf_events diff --git a/tools/testing/selftests/bpf/bpf_kfuncs.h b/tools/testing/selftests/bpf/bpf_kfuncs.h index 5ca68ff0b59fb1..b4e78c1eb37b05 100644 --- a/tools/testing/selftests/bpf/bpf_kfuncs.h +++ b/tools/testing/selftests/bpf/bpf_kfuncs.h @@ -55,4 +55,14 @@ void *bpf_cast_to_kern_ctx(void *) __ksym; void *bpf_rdonly_cast(void *obj, __u32 btf_id) __ksym; +extern int bpf_get_file_xattr(struct file *file, const char *name, + struct bpf_dynptr *value_ptr) __ksym; +extern int bpf_get_fsverity_digest(struct file *file, struct bpf_dynptr *digest_ptr) __ksym; + +extern struct bpf_key *bpf_lookup_user_key(__u32 serial, __u64 flags) __ksym; +extern struct bpf_key *bpf_lookup_system_key(__u64 id) __ksym; +extern void bpf_key_put(struct bpf_key *key) __ksym; +extern int bpf_verify_pkcs7_signature(struct bpf_dynptr *data_ptr, + struct bpf_dynptr *sig_ptr, + struct bpf_key *trusted_keyring) __ksym; #endif diff --git a/tools/testing/selftests/bpf/cgroup_helpers.c b/tools/testing/selftests/bpf/cgroup_helpers.c index 5aa133bf368806..19be9c63d5e84c 100644 --- a/tools/testing/selftests/bpf/cgroup_helpers.c +++ b/tools/testing/selftests/bpf/cgroup_helpers.c @@ -689,3 +689,19 @@ int get_cgroup1_hierarchy_id(const char *subsys_name) fclose(file); return found ? id : -1; } + +/** + * open_classid() - Open a cgroupv1 net_cls classid + * + * This function expects the cgroup work dir to be already created, as we + * open it here. + * + * On success, it returns the file descriptor. On failure it returns -1. + */ +int open_classid(void) +{ + char cgroup_workdir[PATH_MAX + 1]; + + format_classid_path(cgroup_workdir); + return open(cgroup_workdir, O_RDONLY); +} diff --git a/tools/testing/selftests/bpf/cgroup_helpers.h b/tools/testing/selftests/bpf/cgroup_helpers.h index ee053641c0267a..502845160d886a 100644 --- a/tools/testing/selftests/bpf/cgroup_helpers.h +++ b/tools/testing/selftests/bpf/cgroup_helpers.h @@ -33,6 +33,7 @@ void cleanup_cgroup_environment(void); int set_classid(void); int join_classid(void); unsigned long long get_classid_cgroup_id(void); +int open_classid(void); int setup_classid_environment(void); void cleanup_classid_environment(void); diff --git a/tools/testing/selftests/bpf/config b/tools/testing/selftests/bpf/config index 3ec5927ec3e53a..c125c441abc7c8 100644 --- a/tools/testing/selftests/bpf/config +++ b/tools/testing/selftests/bpf/config @@ -23,6 +23,7 @@ CONFIG_FPROBE=y CONFIG_FTRACE_SYSCALLS=y CONFIG_FUNCTION_ERROR_INJECTION=y CONFIG_FUNCTION_TRACER=y +CONFIG_FS_VERITY=y CONFIG_GENEVE=y CONFIG_IKCONFIG=y CONFIG_IKCONFIG_PROC=y @@ -82,7 +83,7 @@ CONFIG_SECURITY=y CONFIG_SECURITYFS=y CONFIG_TEST_BPF=m CONFIG_USERFAULTFD=y +CONFIG_VSOCKETS=y CONFIG_VXLAN=y CONFIG_XDP_SOCKETS=y CONFIG_XFRM_INTERFACE=y -CONFIG_VSOCKETS=y diff --git a/tools/testing/selftests/bpf/prog_tests/btf.c b/tools/testing/selftests/bpf/prog_tests/btf.c index 8fb4a04fbbc042..816145bcb64765 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf.c +++ b/tools/testing/selftests/bpf/prog_tests/btf.c @@ -4630,11 +4630,6 @@ static int test_btf_id(unsigned int test_num) /* The map holds the last ref to BTF and its btf_id */ close(map_fd); map_fd = -1; - btf_fd[0] = bpf_btf_get_fd_by_id(map_info.btf_id); - if (CHECK(btf_fd[0] >= 0, "BTF lingers")) { - err = -1; - goto done; - } fprintf(stderr, "OK"); diff --git a/tools/testing/selftests/bpf/prog_tests/cgrp_local_storage.c b/tools/testing/selftests/bpf/prog_tests/cgrp_local_storage.c index 63e776f4176eb3..747761572098cd 100644 --- a/tools/testing/selftests/bpf/prog_tests/cgrp_local_storage.c +++ b/tools/testing/selftests/bpf/prog_tests/cgrp_local_storage.c @@ -19,6 +19,21 @@ struct socket_cookie { __u64 cookie_value; }; +static bool is_cgroup1; +static int target_hid; + +#define CGROUP_MODE_SET(skel) \ +{ \ + skel->bss->is_cgroup1 = is_cgroup1; \ + skel->bss->target_hid = target_hid; \ +} + +static void cgroup_mode_value_init(bool cgroup, int hid) +{ + is_cgroup1 = cgroup; + target_hid = hid; +} + static void test_tp_btf(int cgroup_fd) { struct cgrp_ls_tp_btf *skel; @@ -29,6 +44,8 @@ static void test_tp_btf(int cgroup_fd) if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) return; + CGROUP_MODE_SET(skel); + /* populate a value in map_b */ err = bpf_map_update_elem(bpf_map__fd(skel->maps.map_b), &cgroup_fd, &val1, BPF_ANY); if (!ASSERT_OK(err, "map_update_elem")) @@ -130,6 +147,8 @@ static void test_recursion(int cgroup_fd) if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) return; + CGROUP_MODE_SET(skel); + err = cgrp_ls_recursion__attach(skel); if (!ASSERT_OK(err, "skel_attach")) goto out; @@ -165,6 +184,8 @@ static void test_cgroup_iter_sleepable(int cgroup_fd, __u64 cgroup_id) if (!ASSERT_OK_PTR(skel, "skel_open")) return; + CGROUP_MODE_SET(skel); + bpf_program__set_autoload(skel->progs.cgroup_iter, true); err = cgrp_ls_sleepable__load(skel); if (!ASSERT_OK(err, "skel_load")) @@ -202,6 +223,7 @@ static void test_yes_rcu_lock(__u64 cgroup_id) if (!ASSERT_OK_PTR(skel, "skel_open")) return; + CGROUP_MODE_SET(skel); skel->bss->target_pid = syscall(SYS_gettid); bpf_program__set_autoload(skel->progs.yes_rcu_lock, true); @@ -229,6 +251,8 @@ static void test_no_rcu_lock(void) if (!ASSERT_OK_PTR(skel, "skel_open")) return; + CGROUP_MODE_SET(skel); + bpf_program__set_autoload(skel->progs.no_rcu_lock, true); err = cgrp_ls_sleepable__load(skel); ASSERT_ERR(err, "skel_load"); @@ -236,7 +260,25 @@ static void test_no_rcu_lock(void) cgrp_ls_sleepable__destroy(skel); } -void test_cgrp_local_storage(void) +static void test_cgrp1_no_rcu_lock(void) +{ + struct cgrp_ls_sleepable *skel; + int err; + + skel = cgrp_ls_sleepable__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + return; + + CGROUP_MODE_SET(skel); + + bpf_program__set_autoload(skel->progs.cgrp1_no_rcu_lock, true); + err = cgrp_ls_sleepable__load(skel); + ASSERT_OK(err, "skel_load"); + + cgrp_ls_sleepable__destroy(skel); +} + +static void cgrp2_local_storage(void) { __u64 cgroup_id; int cgroup_fd; @@ -245,6 +287,8 @@ void test_cgrp_local_storage(void) if (!ASSERT_GE(cgroup_fd, 0, "join_cgroup /cgrp_local_storage")) return; + cgroup_mode_value_init(0, -1); + cgroup_id = get_cgroup_id("/cgrp_local_storage"); if (test__start_subtest("tp_btf")) test_tp_btf(cgroup_fd); @@ -263,3 +307,55 @@ void test_cgrp_local_storage(void) close(cgroup_fd); } + +static void cgrp1_local_storage(void) +{ + int cgrp1_fd, cgrp1_hid, cgrp1_id, err; + + /* Setup cgroup1 hierarchy */ + err = setup_classid_environment(); + if (!ASSERT_OK(err, "setup_classid_environment")) + return; + + err = join_classid(); + if (!ASSERT_OK(err, "join_cgroup1")) + goto cleanup; + + cgrp1_fd = open_classid(); + if (!ASSERT_GE(cgrp1_fd, 0, "cgroup1 fd")) + goto cleanup; + + cgrp1_id = get_classid_cgroup_id(); + if (!ASSERT_GE(cgrp1_id, 0, "cgroup1 id")) + goto close_fd; + + cgrp1_hid = get_cgroup1_hierarchy_id("net_cls"); + if (!ASSERT_GE(cgrp1_hid, 0, "cgroup1 hid")) + goto close_fd; + + cgroup_mode_value_init(1, cgrp1_hid); + + if (test__start_subtest("cgrp1_tp_btf")) + test_tp_btf(cgrp1_fd); + if (test__start_subtest("cgrp1_recursion")) + test_recursion(cgrp1_fd); + if (test__start_subtest("cgrp1_negative")) + test_negative(); + if (test__start_subtest("cgrp1_iter_sleepable")) + test_cgroup_iter_sleepable(cgrp1_fd, cgrp1_id); + if (test__start_subtest("cgrp1_yes_rcu_lock")) + test_yes_rcu_lock(cgrp1_id); + if (test__start_subtest("cgrp1_no_rcu_lock")) + test_cgrp1_no_rcu_lock(); + +close_fd: + close(cgrp1_fd); +cleanup: + cleanup_classid_environment(); +} + +void test_cgrp_local_storage(void) +{ + cgrp2_local_storage(); + cgrp1_local_storage(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/cpumask.c b/tools/testing/selftests/bpf/prog_tests/cpumask.c index 756ea8b590b6fe..c2e886399e3cfb 100644 --- a/tools/testing/selftests/bpf/prog_tests/cpumask.c +++ b/tools/testing/selftests/bpf/prog_tests/cpumask.c @@ -18,6 +18,7 @@ static const char * const cpumask_success_testcases[] = { "test_insert_leave", "test_insert_remove_release", "test_global_mask_rcu", + "test_cpumask_weight", }; static void verify_success(const char *prog_name) diff --git a/tools/testing/selftests/bpf/prog_tests/fs_kfuncs.c b/tools/testing/selftests/bpf/prog_tests/fs_kfuncs.c new file mode 100644 index 00000000000000..37056ba7384743 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/fs_kfuncs.c @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include +#include +#include +#include +#include "test_get_xattr.skel.h" +#include "test_fsverity.skel.h" + +static const char testfile[] = "/tmp/test_progs_fs_kfuncs"; + +static void test_xattr(void) +{ + struct test_get_xattr *skel = NULL; + int fd = -1, err; + + fd = open(testfile, O_CREAT | O_RDONLY, 0644); + if (!ASSERT_GE(fd, 0, "create_file")) + return; + + close(fd); + fd = -1; + + err = setxattr(testfile, "user.kfuncs", "hello", sizeof("hello"), 0); + if (err && errno == EOPNOTSUPP) { + printf("%s:SKIP:local fs doesn't support xattr (%d)\n" + "To run this test, make sure /tmp filesystem supports xattr.\n", + __func__, errno); + test__skip(); + goto out; + } + + if (!ASSERT_OK(err, "setxattr")) + goto out; + + skel = test_get_xattr__open_and_load(); + if (!ASSERT_OK_PTR(skel, "test_get_xattr__open_and_load")) + goto out; + + skel->bss->monitored_pid = getpid(); + err = test_get_xattr__attach(skel); + + if (!ASSERT_OK(err, "test_get_xattr__attach")) + goto out; + + fd = open(testfile, O_RDONLY, 0644); + if (!ASSERT_GE(fd, 0, "open_file")) + goto out; + + ASSERT_EQ(skel->bss->found_xattr, 1, "found_xattr"); + +out: + close(fd); + test_get_xattr__destroy(skel); + remove(testfile); +} + +#ifndef SHA256_DIGEST_SIZE +#define SHA256_DIGEST_SIZE 32 +#endif + +static void test_fsverity(void) +{ + struct fsverity_enable_arg arg = {0}; + struct test_fsverity *skel = NULL; + struct fsverity_digest *d; + int fd, err; + char buffer[4096]; + + fd = open(testfile, O_CREAT | O_RDWR, 0644); + if (!ASSERT_GE(fd, 0, "create_file")) + return; + + /* Write random buffer, so the file is not empty */ + err = write(fd, buffer, 4096); + if (!ASSERT_EQ(err, 4096, "write_file")) + goto out; + close(fd); + + /* Reopen read-only, otherwise FS_IOC_ENABLE_VERITY will fail */ + fd = open(testfile, O_RDONLY, 0644); + if (!ASSERT_GE(fd, 0, "open_file1")) + return; + + /* Enable fsverity for the file. + * If the file system doesn't support verity, this will fail. Skip + * the test in such case. + */ + arg.version = 1; + arg.hash_algorithm = FS_VERITY_HASH_ALG_SHA256; + arg.block_size = 4096; + err = ioctl(fd, FS_IOC_ENABLE_VERITY, &arg); + if (err) { + printf("%s:SKIP:local fs doesn't support fsverity (%d)\n" + "To run this test, try enable CONFIG_FS_VERITY and enable FSVerity for the filesystem.\n", + __func__, errno); + test__skip(); + goto out; + } + + skel = test_fsverity__open_and_load(); + if (!ASSERT_OK_PTR(skel, "test_fsverity__open_and_load")) + goto out; + + /* Get fsverity_digest from ioctl */ + d = (struct fsverity_digest *)skel->bss->expected_digest; + d->digest_algorithm = FS_VERITY_HASH_ALG_SHA256; + d->digest_size = SHA256_DIGEST_SIZE; + err = ioctl(fd, FS_IOC_MEASURE_VERITY, skel->bss->expected_digest); + if (!ASSERT_OK(err, "ioctl_FS_IOC_MEASURE_VERITY")) + goto out; + + skel->bss->monitored_pid = getpid(); + err = test_fsverity__attach(skel); + if (!ASSERT_OK(err, "test_fsverity__attach")) + goto out; + + /* Reopen the file to trigger the program */ + close(fd); + fd = open(testfile, O_RDONLY); + if (!ASSERT_GE(fd, 0, "open_file2")) + goto out; + + ASSERT_EQ(skel->bss->got_fsverity, 1, "got_fsverity"); + ASSERT_EQ(skel->bss->digest_matches, 1, "digest_matches"); +out: + close(fd); + test_fsverity__destroy(skel); + remove(testfile); +} + +void test_fs_kfuncs(void) +{ + if (test__start_subtest("xattr")) + test_xattr(); + + if (test__start_subtest("fsverity")) + test_fsverity(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/global_func_dead_code.c b/tools/testing/selftests/bpf/prog_tests/global_func_dead_code.c new file mode 100644 index 00000000000000..65309894b27acf --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/global_func_dead_code.c @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */ + +#include +#include "verifier_global_subprogs.skel.h" +#include "freplace_dead_global_func.skel.h" + +void test_global_func_dead_code(void) +{ + struct verifier_global_subprogs *tgt_skel = NULL; + struct freplace_dead_global_func *skel = NULL; + char log_buf[4096]; + int err, tgt_fd; + + /* first, try to load target with good global subprog */ + tgt_skel = verifier_global_subprogs__open(); + if (!ASSERT_OK_PTR(tgt_skel, "tgt_skel_good_open")) + return; + + bpf_program__set_autoload(tgt_skel->progs.chained_global_func_calls_success, true); + + err = verifier_global_subprogs__load(tgt_skel); + if (!ASSERT_OK(err, "tgt_skel_good_load")) + goto out; + + tgt_fd = bpf_program__fd(tgt_skel->progs.chained_global_func_calls_success); + + /* Attach to good non-eliminated subprog */ + skel = freplace_dead_global_func__open(); + if (!ASSERT_OK_PTR(skel, "skel_good_open")) + goto out; + + err = bpf_program__set_attach_target(skel->progs.freplace_prog, tgt_fd, "global_good"); + ASSERT_OK(err, "attach_target_good"); + + err = freplace_dead_global_func__load(skel); + if (!ASSERT_OK(err, "skel_good_load")) + goto out; + + freplace_dead_global_func__destroy(skel); + + /* Try attaching to dead code-eliminated subprog */ + skel = freplace_dead_global_func__open(); + if (!ASSERT_OK_PTR(skel, "skel_dead_open")) + goto out; + + bpf_program__set_log_buf(skel->progs.freplace_prog, log_buf, sizeof(log_buf)); + err = bpf_program__set_attach_target(skel->progs.freplace_prog, tgt_fd, "global_dead"); + ASSERT_OK(err, "attach_target_dead"); + + err = freplace_dead_global_func__load(skel); + if (!ASSERT_ERR(err, "skel_dead_load")) + goto out; + + ASSERT_HAS_SUBSTR(log_buf, "Subprog global_dead doesn't exist", "dead_subprog_missing_msg"); + +out: + verifier_global_subprogs__destroy(tgt_skel); + freplace_dead_global_func__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c b/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c index 4041cfa670eb4c..05000810e28ea9 100644 --- a/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c +++ b/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c @@ -222,6 +222,7 @@ static void test_attach_api_fails(void) "bpf_fentry_test2", }; __u64 cookies[2]; + int saved_error; addrs[0] = ksym_get_addr("bpf_fentry_test1"); addrs[1] = ksym_get_addr("bpf_fentry_test2"); @@ -238,10 +239,11 @@ static void test_attach_api_fails(void) /* fail_1 - pattern and opts NULL */ link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual, NULL, NULL); + saved_error = -errno; if (!ASSERT_ERR_PTR(link, "fail_1")) goto cleanup; - if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_1_error")) + if (!ASSERT_EQ(saved_error, -EINVAL, "fail_1_error")) goto cleanup; /* fail_2 - both addrs and syms set */ @@ -252,10 +254,11 @@ static void test_attach_api_fails(void) link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual, NULL, &opts); + saved_error = -errno; if (!ASSERT_ERR_PTR(link, "fail_2")) goto cleanup; - if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_2_error")) + if (!ASSERT_EQ(saved_error, -EINVAL, "fail_2_error")) goto cleanup; /* fail_3 - pattern and addrs set */ @@ -266,10 +269,11 @@ static void test_attach_api_fails(void) link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual, "ksys_*", &opts); + saved_error = -errno; if (!ASSERT_ERR_PTR(link, "fail_3")) goto cleanup; - if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_3_error")) + if (!ASSERT_EQ(saved_error, -EINVAL, "fail_3_error")) goto cleanup; /* fail_4 - pattern and cnt set */ @@ -280,10 +284,11 @@ static void test_attach_api_fails(void) link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual, "ksys_*", &opts); + saved_error = -errno; if (!ASSERT_ERR_PTR(link, "fail_4")) goto cleanup; - if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_4_error")) + if (!ASSERT_EQ(saved_error, -EINVAL, "fail_4_error")) goto cleanup; /* fail_5 - pattern and cookies */ @@ -294,10 +299,26 @@ static void test_attach_api_fails(void) link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual, "ksys_*", &opts); + saved_error = -errno; if (!ASSERT_ERR_PTR(link, "fail_5")) goto cleanup; - if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_5_error")) + if (!ASSERT_EQ(saved_error, -EINVAL, "fail_5_error")) + goto cleanup; + + /* fail_6 - abnormal cnt */ + opts.addrs = (const unsigned long *) addrs; + opts.syms = NULL; + opts.cnt = INT_MAX; + opts.cookies = NULL; + + link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual, + NULL, &opts); + saved_error = -errno; + if (!ASSERT_ERR_PTR(link, "fail_6")) + goto cleanup; + + if (!ASSERT_EQ(saved_error, -E2BIG, "fail_6_error")) goto cleanup; cleanup: diff --git a/tools/testing/selftests/bpf/prog_tests/libbpf_probes.c b/tools/testing/selftests/bpf/prog_tests/libbpf_probes.c index 9f766ddd946aba..4ed46ed58a7b09 100644 --- a/tools/testing/selftests/bpf/prog_tests/libbpf_probes.c +++ b/tools/testing/selftests/bpf/prog_tests/libbpf_probes.c @@ -30,6 +30,8 @@ void test_libbpf_probe_prog_types(void) if (prog_type == BPF_PROG_TYPE_UNSPEC) continue; + if (strcmp(prog_type_name, "__MAX_BPF_PROG_TYPE") == 0) + continue; if (!test__start_subtest(prog_type_name)) continue; @@ -68,6 +70,8 @@ void test_libbpf_probe_map_types(void) if (map_type == BPF_MAP_TYPE_UNSPEC) continue; + if (strcmp(map_type_name, "__MAX_BPF_MAP_TYPE") == 0) + continue; if (!test__start_subtest(map_type_name)) continue; diff --git a/tools/testing/selftests/bpf/prog_tests/libbpf_str.c b/tools/testing/selftests/bpf/prog_tests/libbpf_str.c index c440ea3311ed92..62ea855ec4d04a 100644 --- a/tools/testing/selftests/bpf/prog_tests/libbpf_str.c +++ b/tools/testing/selftests/bpf/prog_tests/libbpf_str.c @@ -87,7 +87,7 @@ static void test_libbpf_bpf_link_type_str(void) const char *link_type_str; char buf[256]; - if (link_type == MAX_BPF_LINK_TYPE) + if (link_type == __MAX_BPF_LINK_TYPE) continue; link_type_name = btf__str_by_offset(btf, e->name_off); @@ -132,6 +132,9 @@ static void test_libbpf_bpf_map_type_str(void) const char *map_type_str; char buf[256]; + if (map_type == __MAX_BPF_MAP_TYPE) + continue; + map_type_name = btf__str_by_offset(btf, e->name_off); map_type_str = libbpf_bpf_map_type_str(map_type); ASSERT_OK_PTR(map_type_str, map_type_name); @@ -186,6 +189,9 @@ static void test_libbpf_bpf_prog_type_str(void) const char *prog_type_str; char buf[256]; + if (prog_type == __MAX_BPF_PROG_TYPE) + continue; + prog_type_name = btf__str_by_offset(btf, e->name_off); prog_type_str = libbpf_bpf_prog_type_str(prog_type); ASSERT_OK_PTR(prog_type_str, prog_type_name); diff --git a/tools/testing/selftests/bpf/prog_tests/local_kptr_stash.c b/tools/testing/selftests/bpf/prog_tests/local_kptr_stash.c index e6e50a394472c2..827e713f6cf1e5 100644 --- a/tools/testing/selftests/bpf/prog_tests/local_kptr_stash.c +++ b/tools/testing/selftests/bpf/prog_tests/local_kptr_stash.c @@ -48,6 +48,27 @@ static void test_local_kptr_stash_plain(void) local_kptr_stash__destroy(skel); } +static void test_local_kptr_stash_local_with_root(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts, + .data_in = &pkt_v4, + .data_size_in = sizeof(pkt_v4), + .repeat = 1, + ); + struct local_kptr_stash *skel; + int ret; + + skel = local_kptr_stash__open_and_load(); + if (!ASSERT_OK_PTR(skel, "local_kptr_stash__open_and_load")) + return; + + ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.stash_local_with_root), &opts); + ASSERT_OK(ret, "local_kptr_stash_add_local_with_root run"); + ASSERT_OK(opts.retval, "local_kptr_stash_add_local_with_root retval"); + + local_kptr_stash__destroy(skel); +} + static void test_local_kptr_stash_unstash(void) { LIBBPF_OPTS(bpf_test_run_opts, opts, @@ -115,6 +136,8 @@ void test_local_kptr_stash(void) test_local_kptr_stash_simple(); if (test__start_subtest("local_kptr_stash_plain")) test_local_kptr_stash_plain(); + if (test__start_subtest("local_kptr_stash_local_with_root")) + test_local_kptr_stash_local_with_root(); if (test__start_subtest("local_kptr_stash_unstash")) test_local_kptr_stash_unstash(); if (test__start_subtest("refcount_acquire_without_unstash")) diff --git a/tools/testing/selftests/bpf/prog_tests/map_btf.c b/tools/testing/selftests/bpf/prog_tests/map_btf.c new file mode 100644 index 00000000000000..2c4ef6037573b6 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/map_btf.c @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2023. Huawei Technologies Co., Ltd */ +#include + +#include "normal_map_btf.skel.h" +#include "map_in_map_btf.skel.h" + +static void do_test_normal_map_btf(void) +{ + struct normal_map_btf *skel; + int i, err, new_fd = -1; + int map_fd_arr[64]; + + skel = normal_map_btf__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_load")) + return; + + err = normal_map_btf__attach(skel); + if (!ASSERT_OK(err, "attach")) + goto out; + + skel->bss->pid = getpid(); + usleep(1); + ASSERT_TRUE(skel->bss->done, "done"); + + /* Use percpu_array to slow bpf_map_free_deferred() down. + * The memory allocation may fail, so doesn't check the returned fd. + */ + for (i = 0; i < ARRAY_SIZE(map_fd_arr); i++) + map_fd_arr[i] = bpf_map_create(BPF_MAP_TYPE_PERCPU_ARRAY, NULL, 4, 4, 256, NULL); + + /* Close array fd later */ + new_fd = dup(bpf_map__fd(skel->maps.array)); +out: + normal_map_btf__destroy(skel); + if (new_fd < 0) + return; + /* Use kern_sync_rcu() to wait for the start of the free of the bpf + * program and use an assumed delay to wait for the release of the map + * btf which is held by other maps (e.g, bss). After that, array map + * holds the last reference of map btf. + */ + kern_sync_rcu(); + usleep(4000); + /* Spawn multiple kworkers to delay the invocation of + * bpf_map_free_deferred() for array map. + */ + for (i = 0; i < ARRAY_SIZE(map_fd_arr); i++) { + if (map_fd_arr[i] < 0) + continue; + close(map_fd_arr[i]); + } + close(new_fd); +} + +static void do_test_map_in_map_btf(void) +{ + int err, zero = 0, new_fd = -1; + struct map_in_map_btf *skel; + + skel = map_in_map_btf__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_load")) + return; + + err = map_in_map_btf__attach(skel); + if (!ASSERT_OK(err, "attach")) + goto out; + + skel->bss->pid = getpid(); + usleep(1); + ASSERT_TRUE(skel->bss->done, "done"); + + /* Close inner_array fd later */ + new_fd = dup(bpf_map__fd(skel->maps.inner_array)); + /* Defer the free of inner_array */ + err = bpf_map__delete_elem(skel->maps.outer_array, &zero, sizeof(zero), 0); + ASSERT_OK(err, "delete inner map"); +out: + map_in_map_btf__destroy(skel); + if (new_fd < 0) + return; + /* Use kern_sync_rcu() to wait for the start of the free of the bpf + * program and use an assumed delay to wait for the free of the outer + * map and the release of map btf. After that, inner map holds the last + * reference of map btf. + */ + kern_sync_rcu(); + usleep(10000); + close(new_fd); +} + +void test_map_btf(void) +{ + if (test__start_subtest("array_btf")) + do_test_normal_map_btf(); + if (test__start_subtest("inner_array_btf")) + do_test_map_in_map_btf(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/map_in_map.c b/tools/testing/selftests/bpf/prog_tests/map_in_map.c new file mode 100644 index 00000000000000..d2a10eb4e5b528 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/map_in_map.c @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2023. Huawei Technologies Co., Ltd */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include "access_map_in_map.skel.h" + +struct thread_ctx { + pthread_barrier_t barrier; + int outer_map_fd; + int start, abort; + int loop, err; +}; + +static int wait_for_start_or_abort(struct thread_ctx *ctx) +{ + while (!ctx->start && !ctx->abort) + usleep(1); + return ctx->abort ? -1 : 0; +} + +static void *update_map_fn(void *data) +{ + struct thread_ctx *ctx = data; + int loop = ctx->loop, err = 0; + + if (wait_for_start_or_abort(ctx) < 0) + return NULL; + pthread_barrier_wait(&ctx->barrier); + + while (loop-- > 0) { + int fd, zero = 0; + + fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, 4, 4, 1, NULL); + if (fd < 0) { + err |= 1; + pthread_barrier_wait(&ctx->barrier); + continue; + } + + /* Remove the old inner map */ + if (bpf_map_update_elem(ctx->outer_map_fd, &zero, &fd, 0) < 0) + err |= 2; + close(fd); + pthread_barrier_wait(&ctx->barrier); + } + + ctx->err = err; + + return NULL; +} + +static void *access_map_fn(void *data) +{ + struct thread_ctx *ctx = data; + int loop = ctx->loop; + + if (wait_for_start_or_abort(ctx) < 0) + return NULL; + pthread_barrier_wait(&ctx->barrier); + + while (loop-- > 0) { + /* Access the old inner map */ + syscall(SYS_getpgid); + pthread_barrier_wait(&ctx->barrier); + } + + return NULL; +} + +static void test_map_in_map_access(const char *prog_name, const char *map_name) +{ + struct access_map_in_map *skel; + struct bpf_map *outer_map; + struct bpf_program *prog; + struct thread_ctx ctx; + pthread_t tid[2]; + int err; + + skel = access_map_in_map__open(); + if (!ASSERT_OK_PTR(skel, "access_map_in_map open")) + return; + + prog = bpf_object__find_program_by_name(skel->obj, prog_name); + if (!ASSERT_OK_PTR(prog, "find program")) + goto out; + bpf_program__set_autoload(prog, true); + + outer_map = bpf_object__find_map_by_name(skel->obj, map_name); + if (!ASSERT_OK_PTR(outer_map, "find map")) + goto out; + + err = access_map_in_map__load(skel); + if (!ASSERT_OK(err, "access_map_in_map load")) + goto out; + + err = access_map_in_map__attach(skel); + if (!ASSERT_OK(err, "access_map_in_map attach")) + goto out; + + skel->bss->tgid = getpid(); + + memset(&ctx, 0, sizeof(ctx)); + pthread_barrier_init(&ctx.barrier, NULL, 2); + ctx.outer_map_fd = bpf_map__fd(outer_map); + ctx.loop = 4; + + err = pthread_create(&tid[0], NULL, update_map_fn, &ctx); + if (!ASSERT_OK(err, "close_thread")) + goto out; + + err = pthread_create(&tid[1], NULL, access_map_fn, &ctx); + if (!ASSERT_OK(err, "read_thread")) { + ctx.abort = 1; + pthread_join(tid[0], NULL); + goto out; + } + + ctx.start = 1; + pthread_join(tid[0], NULL); + pthread_join(tid[1], NULL); + + ASSERT_OK(ctx.err, "err"); +out: + access_map_in_map__destroy(skel); +} + +void test_map_in_map(void) +{ + if (test__start_subtest("acc_map_in_array")) + test_map_in_map_access("access_map_in_array", "outer_array_map"); + if (test__start_subtest("sleepable_acc_map_in_array")) + test_map_in_map_access("sleepable_access_map_in_array", "outer_array_map"); + if (test__start_subtest("acc_map_in_htab")) + test_map_in_map_access("access_map_in_htab", "outer_htab_map"); + if (test__start_subtest("sleepable_acc_map_in_htab")) + test_map_in_map_access("sleepable_access_map_in_htab", "outer_htab_map"); +} + diff --git a/tools/testing/selftests/bpf/prog_tests/syscall.c b/tools/testing/selftests/bpf/prog_tests/syscall.c index f4d40001155a92..0be8301c0ffd25 100644 --- a/tools/testing/selftests/bpf/prog_tests/syscall.c +++ b/tools/testing/selftests/bpf/prog_tests/syscall.c @@ -12,7 +12,7 @@ struct args { int btf_fd; }; -void test_syscall(void) +static void test_syscall_load_prog(void) { static char verifier_log[8192]; struct args ctx = { @@ -32,7 +32,7 @@ void test_syscall(void) if (!ASSERT_OK_PTR(skel, "skel_load")) goto cleanup; - prog_fd = bpf_program__fd(skel->progs.bpf_prog); + prog_fd = bpf_program__fd(skel->progs.load_prog); err = bpf_prog_test_run_opts(prog_fd, &tattr); ASSERT_EQ(err, 0, "err"); ASSERT_EQ(tattr.retval, 1, "retval"); @@ -53,3 +53,29 @@ void test_syscall(void) if (ctx.btf_fd > 0) close(ctx.btf_fd); } + +static void test_syscall_update_outer_map(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct syscall *skel; + int err, prog_fd; + + skel = syscall__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_load")) + goto cleanup; + + prog_fd = bpf_program__fd(skel->progs.update_outer_map); + err = bpf_prog_test_run_opts(prog_fd, &opts); + ASSERT_EQ(err, 0, "err"); + ASSERT_EQ(opts.retval, 1, "retval"); +cleanup: + syscall__destroy(skel); +} + +void test_syscall(void) +{ + if (test__start_subtest("load_prog")) + test_syscall_load_prog(); + if (test__start_subtest("update_outer_map")) + test_syscall_update_outer_map(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/test_tunnel.c b/tools/testing/selftests/bpf/prog_tests/test_tunnel.c index d149ab98798db4..2b3c6dd662590e 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_tunnel.c +++ b/tools/testing/selftests/bpf/prog_tests/test_tunnel.c @@ -50,6 +50,7 @@ */ #include +#include #include #include #include @@ -92,6 +93,11 @@ #define IPIP_TUNL_DEV0 "ipip00" #define IPIP_TUNL_DEV1 "ipip11" +#define XFRM_AUTH "0x1111111111111111111111111111111111111111" +#define XFRM_ENC "0x22222222222222222222222222222222" +#define XFRM_SPI_IN_TO_OUT 0x1 +#define XFRM_SPI_OUT_TO_IN 0x2 + #define PING_ARGS "-i 0.01 -c 3 -w 10 -q" static int config_device(void) @@ -264,6 +270,92 @@ static void delete_ipip_tunnel(void) SYS_NOFAIL("ip fou del port 5555 2> /dev/null"); } +static int add_xfrm_tunnel(void) +{ + /* at_ns0 namespace + * at_ns0 -> root + */ + SYS(fail, + "ip netns exec at_ns0 " + "ip xfrm state add src %s dst %s proto esp " + "spi %d reqid 1 mode tunnel replay-window 42 " + "auth-trunc 'hmac(sha1)' %s 96 enc 'cbc(aes)' %s", + IP4_ADDR_VETH0, IP4_ADDR1_VETH1, XFRM_SPI_IN_TO_OUT, XFRM_AUTH, XFRM_ENC); + SYS(fail, + "ip netns exec at_ns0 " + "ip xfrm policy add src %s/32 dst %s/32 dir out " + "tmpl src %s dst %s proto esp reqid 1 " + "mode tunnel", + IP4_ADDR_TUNL_DEV0, IP4_ADDR_TUNL_DEV1, IP4_ADDR_VETH0, IP4_ADDR1_VETH1); + + /* root -> at_ns0 */ + SYS(fail, + "ip netns exec at_ns0 " + "ip xfrm state add src %s dst %s proto esp " + "spi %d reqid 2 mode tunnel " + "auth-trunc 'hmac(sha1)' %s 96 enc 'cbc(aes)' %s", + IP4_ADDR1_VETH1, IP4_ADDR_VETH0, XFRM_SPI_OUT_TO_IN, XFRM_AUTH, XFRM_ENC); + SYS(fail, + "ip netns exec at_ns0 " + "ip xfrm policy add src %s/32 dst %s/32 dir in " + "tmpl src %s dst %s proto esp reqid 2 " + "mode tunnel", + IP4_ADDR_TUNL_DEV1, IP4_ADDR_TUNL_DEV0, IP4_ADDR1_VETH1, IP4_ADDR_VETH0); + + /* address & route */ + SYS(fail, "ip netns exec at_ns0 ip addr add dev veth0 %s/32", + IP4_ADDR_TUNL_DEV0); + SYS(fail, "ip netns exec at_ns0 ip route add %s dev veth0 via %s src %s", + IP4_ADDR_TUNL_DEV1, IP4_ADDR1_VETH1, IP4_ADDR_TUNL_DEV0); + + /* root namespace + * at_ns0 -> root + */ + SYS(fail, + "ip xfrm state add src %s dst %s proto esp " + "spi %d reqid 1 mode tunnel replay-window 42 " + "auth-trunc 'hmac(sha1)' %s 96 enc 'cbc(aes)' %s", + IP4_ADDR_VETH0, IP4_ADDR1_VETH1, XFRM_SPI_IN_TO_OUT, XFRM_AUTH, XFRM_ENC); + SYS(fail, + "ip xfrm policy add src %s/32 dst %s/32 dir in " + "tmpl src %s dst %s proto esp reqid 1 " + "mode tunnel", + IP4_ADDR_TUNL_DEV0, IP4_ADDR_TUNL_DEV1, IP4_ADDR_VETH0, IP4_ADDR1_VETH1); + + /* root -> at_ns0 */ + SYS(fail, + "ip xfrm state add src %s dst %s proto esp " + "spi %d reqid 2 mode tunnel " + "auth-trunc 'hmac(sha1)' %s 96 enc 'cbc(aes)' %s", + IP4_ADDR1_VETH1, IP4_ADDR_VETH0, XFRM_SPI_OUT_TO_IN, XFRM_AUTH, XFRM_ENC); + SYS(fail, + "ip xfrm policy add src %s/32 dst %s/32 dir out " + "tmpl src %s dst %s proto esp reqid 2 " + "mode tunnel", + IP4_ADDR_TUNL_DEV1, IP4_ADDR_TUNL_DEV0, IP4_ADDR1_VETH1, IP4_ADDR_VETH0); + + /* address & route */ + SYS(fail, "ip addr add dev veth1 %s/32", IP4_ADDR_TUNL_DEV1); + SYS(fail, "ip route add %s dev veth1 via %s src %s", + IP4_ADDR_TUNL_DEV0, IP4_ADDR_VETH0, IP4_ADDR_TUNL_DEV1); + + return 0; +fail: + return -1; +} + +static void delete_xfrm_tunnel(void) +{ + SYS_NOFAIL("ip xfrm policy delete dir out src %s/32 dst %s/32 2> /dev/null", + IP4_ADDR_TUNL_DEV1, IP4_ADDR_TUNL_DEV0); + SYS_NOFAIL("ip xfrm policy delete dir in src %s/32 dst %s/32 2> /dev/null", + IP4_ADDR_TUNL_DEV0, IP4_ADDR_TUNL_DEV1); + SYS_NOFAIL("ip xfrm state delete src %s dst %s proto esp spi %d 2> /dev/null", + IP4_ADDR_VETH0, IP4_ADDR1_VETH1, XFRM_SPI_IN_TO_OUT); + SYS_NOFAIL("ip xfrm state delete src %s dst %s proto esp spi %d 2> /dev/null", + IP4_ADDR1_VETH1, IP4_ADDR_VETH0, XFRM_SPI_OUT_TO_IN); +} + static int test_ping(int family, const char *addr) { SYS(fail, "%s %s %s > /dev/null", ping_command(family), PING_ARGS, addr); @@ -532,25 +624,85 @@ static void test_ipip_tunnel(enum ipip_encap encap) test_tunnel_kern__destroy(skel); } +static void test_xfrm_tunnel(void) +{ + DECLARE_LIBBPF_OPTS(bpf_tc_hook, tc_hook, + .attach_point = BPF_TC_INGRESS); + LIBBPF_OPTS(bpf_xdp_attach_opts, opts); + struct test_tunnel_kern *skel = NULL; + struct nstoken *nstoken; + int xdp_prog_fd; + int tc_prog_fd; + int ifindex; + int err; + + err = add_xfrm_tunnel(); + if (!ASSERT_OK(err, "add_xfrm_tunnel")) + return; + + skel = test_tunnel_kern__open_and_load(); + if (!ASSERT_OK_PTR(skel, "test_tunnel_kern__open_and_load")) + goto done; + + ifindex = if_nametoindex("veth1"); + if (!ASSERT_NEQ(ifindex, 0, "veth1 ifindex")) + goto done; + + /* attach tc prog to tunnel dev */ + tc_hook.ifindex = ifindex; + tc_prog_fd = bpf_program__fd(skel->progs.xfrm_get_state); + if (!ASSERT_GE(tc_prog_fd, 0, "bpf_program__fd")) + goto done; + if (attach_tc_prog(&tc_hook, tc_prog_fd, -1)) + goto done; + + /* attach xdp prog to tunnel dev */ + xdp_prog_fd = bpf_program__fd(skel->progs.xfrm_get_state_xdp); + if (!ASSERT_GE(xdp_prog_fd, 0, "bpf_program__fd")) + goto done; + err = bpf_xdp_attach(ifindex, xdp_prog_fd, XDP_FLAGS_REPLACE, &opts); + if (!ASSERT_OK(err, "bpf_xdp_attach")) + goto done; + + /* ping from at_ns0 namespace test */ + nstoken = open_netns("at_ns0"); + err = test_ping(AF_INET, IP4_ADDR_TUNL_DEV1); + close_netns(nstoken); + if (!ASSERT_OK(err, "test_ping")) + goto done; + + if (!ASSERT_EQ(skel->bss->xfrm_reqid, 1, "req_id")) + goto done; + if (!ASSERT_EQ(skel->bss->xfrm_spi, XFRM_SPI_IN_TO_OUT, "spi")) + goto done; + if (!ASSERT_EQ(skel->bss->xfrm_remote_ip, 0xac100164, "remote_ip")) + goto done; + if (!ASSERT_EQ(skel->bss->xfrm_replay_window, 42, "replay_window")) + goto done; + +done: + delete_xfrm_tunnel(); + if (skel) + test_tunnel_kern__destroy(skel); +} + #define RUN_TEST(name, ...) \ ({ \ if (test__start_subtest(#name)) { \ + config_device(); \ test_ ## name(__VA_ARGS__); \ + cleanup(); \ } \ }) static void *test_tunnel_run_tests(void *arg) { - cleanup(); - config_device(); - RUN_TEST(vxlan_tunnel); RUN_TEST(ip6vxlan_tunnel); RUN_TEST(ipip_tunnel, NONE); RUN_TEST(ipip_tunnel, FOU); RUN_TEST(ipip_tunnel, GUE); - - cleanup(); + RUN_TEST(xfrm_tunnel); return NULL; } diff --git a/tools/testing/selftests/bpf/prog_tests/time_tai.c b/tools/testing/selftests/bpf/prog_tests/time_tai.c index a311198236661b..f45af1b0ef2c44 100644 --- a/tools/testing/selftests/bpf/prog_tests/time_tai.c +++ b/tools/testing/selftests/bpf/prog_tests/time_tai.c @@ -56,7 +56,7 @@ void test_time_tai(void) ASSERT_NEQ(ts2, 0, "tai_ts2"); /* TAI is moving forward only */ - ASSERT_GT(ts2, ts1, "tai_forward"); + ASSERT_GE(ts2, ts1, "tai_forward"); /* Check for future */ ret = clock_gettime(CLOCK_TAI, &now_tai); diff --git a/tools/testing/selftests/bpf/prog_tests/token.c b/tools/testing/selftests/bpf/prog_tests/token.c new file mode 100644 index 00000000000000..b5dce630e0e138 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/token.c @@ -0,0 +1,1031 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */ +#define _GNU_SOURCE +#include +#include +#include "cap_helpers.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "priv_map.skel.h" +#include "priv_prog.skel.h" +#include "dummy_st_ops_success.skel.h" + +static inline int sys_mount(const char *dev_name, const char *dir_name, + const char *type, unsigned long flags, + const void *data) +{ + return syscall(__NR_mount, dev_name, dir_name, type, flags, data); +} + +static inline int sys_fsopen(const char *fsname, unsigned flags) +{ + return syscall(__NR_fsopen, fsname, flags); +} + +static inline int sys_fspick(int dfd, const char *path, unsigned flags) +{ + return syscall(__NR_fspick, dfd, path, flags); +} + +static inline int sys_fsconfig(int fs_fd, unsigned cmd, const char *key, const void *val, int aux) +{ + return syscall(__NR_fsconfig, fs_fd, cmd, key, val, aux); +} + +static inline int sys_fsmount(int fs_fd, unsigned flags, unsigned ms_flags) +{ + return syscall(__NR_fsmount, fs_fd, flags, ms_flags); +} + +static inline int sys_move_mount(int from_dfd, const char *from_path, + int to_dfd, const char *to_path, + unsigned flags) +{ + return syscall(__NR_move_mount, from_dfd, from_path, to_dfd, to_path, flags); +} + +static int drop_priv_caps(__u64 *old_caps) +{ + return cap_disable_effective((1ULL << CAP_BPF) | + (1ULL << CAP_PERFMON) | + (1ULL << CAP_NET_ADMIN) | + (1ULL << CAP_SYS_ADMIN), old_caps); +} + +static int restore_priv_caps(__u64 old_caps) +{ + return cap_enable_effective(old_caps, NULL); +} + +static int set_delegate_mask(int fs_fd, const char *key, __u64 mask, const char *mask_str) +{ + char buf[32]; + int err; + + if (!mask_str) { + if (mask == ~0ULL) { + mask_str = "any"; + } else { + snprintf(buf, sizeof(buf), "0x%llx", (unsigned long long)mask); + mask_str = buf; + } + } + + err = sys_fsconfig(fs_fd, FSCONFIG_SET_STRING, key, + mask_str, 0); + if (err < 0) + err = -errno; + return err; +} + +#define zclose(fd) do { if (fd >= 0) close(fd); fd = -1; } while (0) + +struct bpffs_opts { + __u64 cmds; + __u64 maps; + __u64 progs; + __u64 attachs; + const char *cmds_str; + const char *maps_str; + const char *progs_str; + const char *attachs_str; +}; + +static int create_bpffs_fd(void) +{ + int fs_fd; + + /* create VFS context */ + fs_fd = sys_fsopen("bpf", 0); + ASSERT_GE(fs_fd, 0, "fs_fd"); + + return fs_fd; +} + +static int materialize_bpffs_fd(int fs_fd, struct bpffs_opts *opts) +{ + int mnt_fd, err; + + /* set up token delegation mount options */ + err = set_delegate_mask(fs_fd, "delegate_cmds", opts->cmds, opts->cmds_str); + if (!ASSERT_OK(err, "fs_cfg_cmds")) + return err; + err = set_delegate_mask(fs_fd, "delegate_maps", opts->maps, opts->maps_str); + if (!ASSERT_OK(err, "fs_cfg_maps")) + return err; + err = set_delegate_mask(fs_fd, "delegate_progs", opts->progs, opts->progs_str); + if (!ASSERT_OK(err, "fs_cfg_progs")) + return err; + err = set_delegate_mask(fs_fd, "delegate_attachs", opts->attachs, opts->attachs_str); + if (!ASSERT_OK(err, "fs_cfg_attachs")) + return err; + + /* instantiate FS object */ + err = sys_fsconfig(fs_fd, FSCONFIG_CMD_CREATE, NULL, NULL, 0); + if (err < 0) + return -errno; + + /* create O_PATH fd for detached mount */ + mnt_fd = sys_fsmount(fs_fd, 0, 0); + if (err < 0) + return -errno; + + return mnt_fd; +} + +/* send FD over Unix domain (AF_UNIX) socket */ +static int sendfd(int sockfd, int fd) +{ + struct msghdr msg = {}; + struct cmsghdr *cmsg; + int fds[1] = { fd }, err; + char iobuf[1]; + struct iovec io = { + .iov_base = iobuf, + .iov_len = sizeof(iobuf), + }; + union { + char buf[CMSG_SPACE(sizeof(fds))]; + struct cmsghdr align; + } u; + + msg.msg_iov = &io; + msg.msg_iovlen = 1; + msg.msg_control = u.buf; + msg.msg_controllen = sizeof(u.buf); + cmsg = CMSG_FIRSTHDR(&msg); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_RIGHTS; + cmsg->cmsg_len = CMSG_LEN(sizeof(fds)); + memcpy(CMSG_DATA(cmsg), fds, sizeof(fds)); + + err = sendmsg(sockfd, &msg, 0); + if (err < 0) + err = -errno; + if (!ASSERT_EQ(err, 1, "sendmsg")) + return -EINVAL; + + return 0; +} + +/* receive FD over Unix domain (AF_UNIX) socket */ +static int recvfd(int sockfd, int *fd) +{ + struct msghdr msg = {}; + struct cmsghdr *cmsg; + int fds[1], err; + char iobuf[1]; + struct iovec io = { + .iov_base = iobuf, + .iov_len = sizeof(iobuf), + }; + union { + char buf[CMSG_SPACE(sizeof(fds))]; + struct cmsghdr align; + } u; + + msg.msg_iov = &io; + msg.msg_iovlen = 1; + msg.msg_control = u.buf; + msg.msg_controllen = sizeof(u.buf); + + err = recvmsg(sockfd, &msg, 0); + if (err < 0) + err = -errno; + if (!ASSERT_EQ(err, 1, "recvmsg")) + return -EINVAL; + + cmsg = CMSG_FIRSTHDR(&msg); + if (!ASSERT_OK_PTR(cmsg, "cmsg_null") || + !ASSERT_EQ(cmsg->cmsg_len, CMSG_LEN(sizeof(fds)), "cmsg_len") || + !ASSERT_EQ(cmsg->cmsg_level, SOL_SOCKET, "cmsg_level") || + !ASSERT_EQ(cmsg->cmsg_type, SCM_RIGHTS, "cmsg_type")) + return -EINVAL; + + memcpy(fds, CMSG_DATA(cmsg), sizeof(fds)); + *fd = fds[0]; + + return 0; +} + +static ssize_t write_nointr(int fd, const void *buf, size_t count) +{ + ssize_t ret; + + do { + ret = write(fd, buf, count); + } while (ret < 0 && errno == EINTR); + + return ret; +} + +static int write_file(const char *path, const void *buf, size_t count) +{ + int fd; + ssize_t ret; + + fd = open(path, O_WRONLY | O_CLOEXEC | O_NOCTTY | O_NOFOLLOW); + if (fd < 0) + return -1; + + ret = write_nointr(fd, buf, count); + close(fd); + if (ret < 0 || (size_t)ret != count) + return -1; + + return 0; +} + +static int create_and_enter_userns(void) +{ + uid_t uid; + gid_t gid; + char map[100]; + + uid = getuid(); + gid = getgid(); + + if (unshare(CLONE_NEWUSER)) + return -1; + + if (write_file("/proc/self/setgroups", "deny", sizeof("deny") - 1) && + errno != ENOENT) + return -1; + + snprintf(map, sizeof(map), "0 %d 1", uid); + if (write_file("/proc/self/uid_map", map, strlen(map))) + return -1; + + + snprintf(map, sizeof(map), "0 %d 1", gid); + if (write_file("/proc/self/gid_map", map, strlen(map))) + return -1; + + if (setgid(0)) + return -1; + + if (setuid(0)) + return -1; + + return 0; +} + +typedef int (*child_callback_fn)(int); + +static void child(int sock_fd, struct bpffs_opts *opts, child_callback_fn callback) +{ + LIBBPF_OPTS(bpf_map_create_opts, map_opts); + int mnt_fd = -1, fs_fd = -1, err = 0, bpffs_fd = -1; + + /* setup userns with root mappings */ + err = create_and_enter_userns(); + if (!ASSERT_OK(err, "create_and_enter_userns")) + goto cleanup; + + /* setup mountns to allow creating BPF FS (fsopen("bpf")) from unpriv process */ + err = unshare(CLONE_NEWNS); + if (!ASSERT_OK(err, "create_mountns")) + goto cleanup; + + err = sys_mount(NULL, "/", NULL, MS_REC | MS_PRIVATE, 0); + if (!ASSERT_OK(err, "remount_root")) + goto cleanup; + + fs_fd = create_bpffs_fd(); + if (!ASSERT_GE(fs_fd, 0, "create_bpffs_fd")) { + err = -EINVAL; + goto cleanup; + } + + /* ensure unprivileged child cannot set delegation options */ + err = set_delegate_mask(fs_fd, "delegate_cmds", 0x1, NULL); + ASSERT_EQ(err, -EPERM, "delegate_cmd_eperm"); + err = set_delegate_mask(fs_fd, "delegate_maps", 0x1, NULL); + ASSERT_EQ(err, -EPERM, "delegate_maps_eperm"); + err = set_delegate_mask(fs_fd, "delegate_progs", 0x1, NULL); + ASSERT_EQ(err, -EPERM, "delegate_progs_eperm"); + err = set_delegate_mask(fs_fd, "delegate_attachs", 0x1, NULL); + ASSERT_EQ(err, -EPERM, "delegate_attachs_eperm"); + + /* pass BPF FS context object to parent */ + err = sendfd(sock_fd, fs_fd); + if (!ASSERT_OK(err, "send_fs_fd")) + goto cleanup; + zclose(fs_fd); + + /* avoid mucking around with mount namespaces and mounting at + * well-known path, just get detach-mounted BPF FS fd back from parent + */ + err = recvfd(sock_fd, &mnt_fd); + if (!ASSERT_OK(err, "recv_mnt_fd")) + goto cleanup; + + /* try to fspick() BPF FS and try to add some delegation options */ + fs_fd = sys_fspick(mnt_fd, "", FSPICK_EMPTY_PATH); + if (!ASSERT_GE(fs_fd, 0, "bpffs_fspick")) { + err = -EINVAL; + goto cleanup; + } + + /* ensure unprivileged child cannot reconfigure to set delegation options */ + err = set_delegate_mask(fs_fd, "delegate_cmds", 0, "any"); + if (!ASSERT_EQ(err, -EPERM, "delegate_cmd_eperm_reconfig")) { + err = -EINVAL; + goto cleanup; + } + err = set_delegate_mask(fs_fd, "delegate_maps", 0, "any"); + if (!ASSERT_EQ(err, -EPERM, "delegate_maps_eperm_reconfig")) { + err = -EINVAL; + goto cleanup; + } + err = set_delegate_mask(fs_fd, "delegate_progs", 0, "any"); + if (!ASSERT_EQ(err, -EPERM, "delegate_progs_eperm_reconfig")) { + err = -EINVAL; + goto cleanup; + } + err = set_delegate_mask(fs_fd, "delegate_attachs", 0, "any"); + if (!ASSERT_EQ(err, -EPERM, "delegate_attachs_eperm_reconfig")) { + err = -EINVAL; + goto cleanup; + } + zclose(fs_fd); + + bpffs_fd = openat(mnt_fd, ".", 0, O_RDWR); + if (!ASSERT_GE(bpffs_fd, 0, "bpffs_open")) { + err = -EINVAL; + goto cleanup; + } + + /* do custom test logic with customly set up BPF FS instance */ + err = callback(bpffs_fd); + if (!ASSERT_OK(err, "test_callback")) + goto cleanup; + + err = 0; +cleanup: + zclose(sock_fd); + zclose(mnt_fd); + zclose(fs_fd); + zclose(bpffs_fd); + + exit(-err); +} + +static int wait_for_pid(pid_t pid) +{ + int status, ret; + +again: + ret = waitpid(pid, &status, 0); + if (ret == -1) { + if (errno == EINTR) + goto again; + + return -1; + } + + if (!WIFEXITED(status)) + return -1; + + return WEXITSTATUS(status); +} + +static void parent(int child_pid, struct bpffs_opts *bpffs_opts, int sock_fd) +{ + int fs_fd = -1, mnt_fd = -1, err; + + err = recvfd(sock_fd, &fs_fd); + if (!ASSERT_OK(err, "recv_bpffs_fd")) + goto cleanup; + + mnt_fd = materialize_bpffs_fd(fs_fd, bpffs_opts); + if (!ASSERT_GE(mnt_fd, 0, "materialize_bpffs_fd")) { + err = -EINVAL; + goto cleanup; + } + zclose(fs_fd); + + /* pass BPF FS context object to parent */ + err = sendfd(sock_fd, mnt_fd); + if (!ASSERT_OK(err, "send_mnt_fd")) + goto cleanup; + zclose(mnt_fd); + + err = wait_for_pid(child_pid); + ASSERT_OK(err, "waitpid_child"); + +cleanup: + zclose(sock_fd); + zclose(fs_fd); + zclose(mnt_fd); + + if (child_pid > 0) + (void)kill(child_pid, SIGKILL); +} + +static void subtest_userns(struct bpffs_opts *bpffs_opts, child_callback_fn cb) +{ + int sock_fds[2] = { -1, -1 }; + int child_pid = 0, err; + + err = socketpair(AF_UNIX, SOCK_STREAM, 0, sock_fds); + if (!ASSERT_OK(err, "socketpair")) + goto cleanup; + + child_pid = fork(); + if (!ASSERT_GE(child_pid, 0, "fork")) + goto cleanup; + + if (child_pid == 0) { + zclose(sock_fds[0]); + return child(sock_fds[1], bpffs_opts, cb); + + } else { + zclose(sock_fds[1]); + return parent(child_pid, bpffs_opts, sock_fds[0]); + } + +cleanup: + zclose(sock_fds[0]); + zclose(sock_fds[1]); + if (child_pid > 0) + (void)kill(child_pid, SIGKILL); +} + +static int userns_map_create(int mnt_fd) +{ + LIBBPF_OPTS(bpf_map_create_opts, map_opts); + int err, token_fd = -1, map_fd = -1; + __u64 old_caps = 0; + + /* create BPF token from BPF FS mount */ + token_fd = bpf_token_create(mnt_fd, NULL); + if (!ASSERT_GT(token_fd, 0, "token_create")) { + err = -EINVAL; + goto cleanup; + } + + /* while inside non-init userns, we need both a BPF token *and* + * CAP_BPF inside current userns to create privileged map; let's test + * that neither BPF token alone nor namespaced CAP_BPF is sufficient + */ + err = drop_priv_caps(&old_caps); + if (!ASSERT_OK(err, "drop_caps")) + goto cleanup; + + /* no token, no CAP_BPF -> fail */ + map_opts.token_fd = 0; + map_fd = bpf_map_create(BPF_MAP_TYPE_STACK, "wo_token_wo_bpf", 0, 8, 1, &map_opts); + if (!ASSERT_LT(map_fd, 0, "stack_map_wo_token_wo_cap_bpf_should_fail")) { + err = -EINVAL; + goto cleanup; + } + + /* token without CAP_BPF -> fail */ + map_opts.token_fd = token_fd; + map_fd = bpf_map_create(BPF_MAP_TYPE_STACK, "w_token_wo_bpf", 0, 8, 1, &map_opts); + if (!ASSERT_LT(map_fd, 0, "stack_map_w_token_wo_cap_bpf_should_fail")) { + err = -EINVAL; + goto cleanup; + } + + /* get back effective local CAP_BPF (and CAP_SYS_ADMIN) */ + err = restore_priv_caps(old_caps); + if (!ASSERT_OK(err, "restore_caps")) + goto cleanup; + + /* CAP_BPF without token -> fail */ + map_opts.token_fd = 0; + map_fd = bpf_map_create(BPF_MAP_TYPE_STACK, "wo_token_w_bpf", 0, 8, 1, &map_opts); + if (!ASSERT_LT(map_fd, 0, "stack_map_wo_token_w_cap_bpf_should_fail")) { + err = -EINVAL; + goto cleanup; + } + + /* finally, namespaced CAP_BPF + token -> success */ + map_opts.token_fd = token_fd; + map_fd = bpf_map_create(BPF_MAP_TYPE_STACK, "w_token_w_bpf", 0, 8, 1, &map_opts); + if (!ASSERT_GT(map_fd, 0, "stack_map_w_token_w_cap_bpf")) { + err = -EINVAL; + goto cleanup; + } + +cleanup: + zclose(token_fd); + zclose(map_fd); + return err; +} + +static int userns_btf_load(int mnt_fd) +{ + LIBBPF_OPTS(bpf_btf_load_opts, btf_opts); + int err, token_fd = -1, btf_fd = -1; + const void *raw_btf_data; + struct btf *btf = NULL; + __u32 raw_btf_size; + __u64 old_caps = 0; + + /* create BPF token from BPF FS mount */ + token_fd = bpf_token_create(mnt_fd, NULL); + if (!ASSERT_GT(token_fd, 0, "token_create")) { + err = -EINVAL; + goto cleanup; + } + + /* while inside non-init userns, we need both a BPF token *and* + * CAP_BPF inside current userns to create privileged map; let's test + * that neither BPF token alone nor namespaced CAP_BPF is sufficient + */ + err = drop_priv_caps(&old_caps); + if (!ASSERT_OK(err, "drop_caps")) + goto cleanup; + + /* setup a trivial BTF data to load to the kernel */ + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "empty_btf")) + goto cleanup; + + ASSERT_GT(btf__add_int(btf, "int", 4, 0), 0, "int_type"); + + raw_btf_data = btf__raw_data(btf, &raw_btf_size); + if (!ASSERT_OK_PTR(raw_btf_data, "raw_btf_data")) + goto cleanup; + + /* no token + no CAP_BPF -> failure */ + btf_opts.token_fd = 0; + btf_fd = bpf_btf_load(raw_btf_data, raw_btf_size, &btf_opts); + if (!ASSERT_LT(btf_fd, 0, "no_token_no_cap_should_fail")) + goto cleanup; + + /* token + no CAP_BPF -> failure */ + btf_opts.token_fd = token_fd; + btf_fd = bpf_btf_load(raw_btf_data, raw_btf_size, &btf_opts); + if (!ASSERT_LT(btf_fd, 0, "token_no_cap_should_fail")) + goto cleanup; + + /* get back effective local CAP_BPF (and CAP_SYS_ADMIN) */ + err = restore_priv_caps(old_caps); + if (!ASSERT_OK(err, "restore_caps")) + goto cleanup; + + /* token + CAP_BPF -> success */ + btf_opts.token_fd = token_fd; + btf_fd = bpf_btf_load(raw_btf_data, raw_btf_size, &btf_opts); + if (!ASSERT_GT(btf_fd, 0, "token_and_cap_success")) + goto cleanup; + + err = 0; +cleanup: + btf__free(btf); + zclose(btf_fd); + zclose(token_fd); + return err; +} + +static int userns_prog_load(int mnt_fd) +{ + LIBBPF_OPTS(bpf_prog_load_opts, prog_opts); + int err, token_fd = -1, prog_fd = -1; + struct bpf_insn insns[] = { + /* bpf_jiffies64() requires CAP_BPF */ + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_jiffies64), + /* bpf_get_current_task() requires CAP_PERFMON */ + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_get_current_task), + /* r0 = 0; exit; */ + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + size_t insn_cnt = ARRAY_SIZE(insns); + __u64 old_caps = 0; + + /* create BPF token from BPF FS mount */ + token_fd = bpf_token_create(mnt_fd, NULL); + if (!ASSERT_GT(token_fd, 0, "token_create")) { + err = -EINVAL; + goto cleanup; + } + + /* validate we can successfully load BPF program with token; this + * being XDP program (CAP_NET_ADMIN) using bpf_jiffies64() (CAP_BPF) + * and bpf_get_current_task() (CAP_PERFMON) helpers validates we have + * BPF token wired properly in a bunch of places in the kernel + */ + prog_opts.token_fd = token_fd; + prog_opts.expected_attach_type = BPF_XDP; + prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, "token_prog", "GPL", + insns, insn_cnt, &prog_opts); + if (!ASSERT_GT(prog_fd, 0, "prog_fd")) { + err = -EPERM; + goto cleanup; + } + + /* no token + caps -> failure */ + prog_opts.token_fd = 0; + prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, "token_prog", "GPL", + insns, insn_cnt, &prog_opts); + if (!ASSERT_EQ(prog_fd, -EPERM, "prog_fd_eperm")) { + err = -EPERM; + goto cleanup; + } + + err = drop_priv_caps(&old_caps); + if (!ASSERT_OK(err, "drop_caps")) + goto cleanup; + + /* no caps + token -> failure */ + prog_opts.token_fd = token_fd; + prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, "token_prog", "GPL", + insns, insn_cnt, &prog_opts); + if (!ASSERT_EQ(prog_fd, -EPERM, "prog_fd_eperm")) { + err = -EPERM; + goto cleanup; + } + + /* no caps + no token -> definitely a failure */ + prog_opts.token_fd = 0; + prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, "token_prog", "GPL", + insns, insn_cnt, &prog_opts); + if (!ASSERT_EQ(prog_fd, -EPERM, "prog_fd_eperm")) { + err = -EPERM; + goto cleanup; + } + + err = 0; +cleanup: + zclose(prog_fd); + zclose(token_fd); + return err; +} + +static int userns_obj_priv_map(int mnt_fd) +{ + LIBBPF_OPTS(bpf_object_open_opts, opts); + char buf[256]; + struct priv_map *skel; + int err, token_fd; + + skel = priv_map__open_and_load(); + if (!ASSERT_ERR_PTR(skel, "obj_tokenless_load")) { + priv_map__destroy(skel); + return -EINVAL; + } + + /* use bpf_token_path to provide BPF FS path */ + snprintf(buf, sizeof(buf), "/proc/self/fd/%d", mnt_fd); + opts.bpf_token_path = buf; + skel = priv_map__open_opts(&opts); + if (!ASSERT_OK_PTR(skel, "obj_token_path_open")) + return -EINVAL; + + err = priv_map__load(skel); + priv_map__destroy(skel); + if (!ASSERT_OK(err, "obj_token_path_load")) + return -EINVAL; + + /* create token and pass it through bpf_token_fd */ + token_fd = bpf_token_create(mnt_fd, NULL); + if (!ASSERT_GT(token_fd, 0, "create_token")) + return -EINVAL; + + opts.bpf_token_path = NULL; + opts.bpf_token_fd = token_fd; + skel = priv_map__open_opts(&opts); + if (!ASSERT_OK_PTR(skel, "obj_token_fd_open")) + return -EINVAL; + + /* we can close our token FD, bpf_object owns dup()'ed FD now */ + close(token_fd); + + err = priv_map__load(skel); + priv_map__destroy(skel); + if (!ASSERT_OK(err, "obj_token_fd_load")) + return -EINVAL; + + return 0; +} + +static int userns_obj_priv_prog(int mnt_fd) +{ + LIBBPF_OPTS(bpf_object_open_opts, opts); + char buf[256]; + struct priv_prog *skel; + int err; + + skel = priv_prog__open_and_load(); + if (!ASSERT_ERR_PTR(skel, "obj_tokenless_load")) { + priv_prog__destroy(skel); + return -EINVAL; + } + + /* use bpf_token_path to provide BPF FS path */ + snprintf(buf, sizeof(buf), "/proc/self/fd/%d", mnt_fd); + opts.bpf_token_path = buf; + skel = priv_prog__open_opts(&opts); + if (!ASSERT_OK_PTR(skel, "obj_token_path_open")) + return -EINVAL; + + err = priv_prog__load(skel); + priv_prog__destroy(skel); + if (!ASSERT_OK(err, "obj_token_path_load")) + return -EINVAL; + + return 0; +} + +/* this test is called with BPF FS that doesn't delegate BPF_BTF_LOAD command, + * which should cause struct_ops application to fail, as BTF won't be uploaded + * into the kernel, even if STRUCT_OPS programs themselves are allowed + */ +static int validate_struct_ops_load(int mnt_fd, bool expect_success) +{ + LIBBPF_OPTS(bpf_object_open_opts, opts); + char buf[256]; + struct dummy_st_ops_success *skel; + int err; + + snprintf(buf, sizeof(buf), "/proc/self/fd/%d", mnt_fd); + opts.bpf_token_path = buf; + skel = dummy_st_ops_success__open_opts(&opts); + if (!ASSERT_OK_PTR(skel, "obj_token_path_open")) + return -EINVAL; + + err = dummy_st_ops_success__load(skel); + dummy_st_ops_success__destroy(skel); + if (expect_success) { + if (!ASSERT_OK(err, "obj_token_path_load")) + return -EINVAL; + } else /* expect failure */ { + if (!ASSERT_ERR(err, "obj_token_path_load")) + return -EINVAL; + } + + return 0; +} + +static int userns_obj_priv_btf_fail(int mnt_fd) +{ + return validate_struct_ops_load(mnt_fd, false /* should fail */); +} + +static int userns_obj_priv_btf_success(int mnt_fd) +{ + return validate_struct_ops_load(mnt_fd, true /* should succeed */); +} + +#define TOKEN_ENVVAR "LIBBPF_BPF_TOKEN_PATH" +#define TOKEN_BPFFS_CUSTOM "/bpf-token-fs" + +static int userns_obj_priv_implicit_token(int mnt_fd) +{ + LIBBPF_OPTS(bpf_object_open_opts, opts); + struct dummy_st_ops_success *skel; + int err; + + /* before we mount BPF FS with token delegation, struct_ops skeleton + * should fail to load + */ + skel = dummy_st_ops_success__open_and_load(); + if (!ASSERT_ERR_PTR(skel, "obj_tokenless_load")) { + dummy_st_ops_success__destroy(skel); + return -EINVAL; + } + + /* mount custom BPF FS over /sys/fs/bpf so that libbpf can create BPF + * token automatically and implicitly + */ + err = sys_move_mount(mnt_fd, "", AT_FDCWD, "/sys/fs/bpf", MOVE_MOUNT_F_EMPTY_PATH); + if (!ASSERT_OK(err, "move_mount_bpffs")) + return -EINVAL; + + /* disable implicit BPF token creation by setting + * LIBBPF_BPF_TOKEN_PATH envvar to empty value, load should fail + */ + err = setenv(TOKEN_ENVVAR, "", 1 /*overwrite*/); + if (!ASSERT_OK(err, "setenv_token_path")) + return -EINVAL; + skel = dummy_st_ops_success__open_and_load(); + if (!ASSERT_ERR_PTR(skel, "obj_token_envvar_disabled_load")) { + unsetenv(TOKEN_ENVVAR); + dummy_st_ops_success__destroy(skel); + return -EINVAL; + } + unsetenv(TOKEN_ENVVAR); + + /* now the same struct_ops skeleton should succeed thanks to libppf + * creating BPF token from /sys/fs/bpf mount point + */ + skel = dummy_st_ops_success__open_and_load(); + if (!ASSERT_OK_PTR(skel, "obj_implicit_token_load")) + return -EINVAL; + + dummy_st_ops_success__destroy(skel); + + /* now disable implicit token through empty bpf_token_path, should fail */ + opts.bpf_token_path = ""; + skel = dummy_st_ops_success__open_opts(&opts); + if (!ASSERT_OK_PTR(skel, "obj_empty_token_path_open")) + return -EINVAL; + + err = dummy_st_ops_success__load(skel); + dummy_st_ops_success__destroy(skel); + if (!ASSERT_ERR(err, "obj_empty_token_path_load")) + return -EINVAL; + + /* now disable implicit token through negative bpf_token_fd, should fail */ + opts.bpf_token_path = NULL; + opts.bpf_token_fd = -1; + skel = dummy_st_ops_success__open_opts(&opts); + if (!ASSERT_OK_PTR(skel, "obj_neg_token_fd_open")) + return -EINVAL; + + err = dummy_st_ops_success__load(skel); + dummy_st_ops_success__destroy(skel); + if (!ASSERT_ERR(err, "obj_neg_token_fd_load")) + return -EINVAL; + + return 0; +} + +static int userns_obj_priv_implicit_token_envvar(int mnt_fd) +{ + LIBBPF_OPTS(bpf_object_open_opts, opts); + struct dummy_st_ops_success *skel; + int err; + + /* before we mount BPF FS with token delegation, struct_ops skeleton + * should fail to load + */ + skel = dummy_st_ops_success__open_and_load(); + if (!ASSERT_ERR_PTR(skel, "obj_tokenless_load")) { + dummy_st_ops_success__destroy(skel); + return -EINVAL; + } + + /* mount custom BPF FS over custom location, so libbpf can't create + * BPF token implicitly, unless pointed to it through + * LIBBPF_BPF_TOKEN_PATH envvar + */ + rmdir(TOKEN_BPFFS_CUSTOM); + if (!ASSERT_OK(mkdir(TOKEN_BPFFS_CUSTOM, 0777), "mkdir_bpffs_custom")) + goto err_out; + err = sys_move_mount(mnt_fd, "", AT_FDCWD, TOKEN_BPFFS_CUSTOM, MOVE_MOUNT_F_EMPTY_PATH); + if (!ASSERT_OK(err, "move_mount_bpffs")) + goto err_out; + + /* even though we have BPF FS with delegation, it's not at default + * /sys/fs/bpf location, so we still fail to load until envvar is set up + */ + skel = dummy_st_ops_success__open_and_load(); + if (!ASSERT_ERR_PTR(skel, "obj_tokenless_load2")) { + dummy_st_ops_success__destroy(skel); + goto err_out; + } + + err = setenv(TOKEN_ENVVAR, TOKEN_BPFFS_CUSTOM, 1 /*overwrite*/); + if (!ASSERT_OK(err, "setenv_token_path")) + goto err_out; + + /* now the same struct_ops skeleton should succeed thanks to libppf + * creating BPF token from custom mount point + */ + skel = dummy_st_ops_success__open_and_load(); + if (!ASSERT_OK_PTR(skel, "obj_implicit_token_load")) + goto err_out; + + dummy_st_ops_success__destroy(skel); + + /* now disable implicit token through empty bpf_token_path, envvar + * will be ignored, should fail + */ + opts.bpf_token_path = ""; + skel = dummy_st_ops_success__open_opts(&opts); + if (!ASSERT_OK_PTR(skel, "obj_empty_token_path_open")) + goto err_out; + + err = dummy_st_ops_success__load(skel); + dummy_st_ops_success__destroy(skel); + if (!ASSERT_ERR(err, "obj_empty_token_path_load")) + goto err_out; + + /* now disable implicit token through negative bpf_token_fd, envvar + * will be ignored, should fail + */ + opts.bpf_token_path = NULL; + opts.bpf_token_fd = -1; + skel = dummy_st_ops_success__open_opts(&opts); + if (!ASSERT_OK_PTR(skel, "obj_neg_token_fd_open")) + goto err_out; + + err = dummy_st_ops_success__load(skel); + dummy_st_ops_success__destroy(skel); + if (!ASSERT_ERR(err, "obj_neg_token_fd_load")) + goto err_out; + + rmdir(TOKEN_BPFFS_CUSTOM); + unsetenv(TOKEN_ENVVAR); + return 0; +err_out: + rmdir(TOKEN_BPFFS_CUSTOM); + unsetenv(TOKEN_ENVVAR); + return -EINVAL; +} + +#define bit(n) (1ULL << (n)) + +void test_token(void) +{ + if (test__start_subtest("map_token")) { + struct bpffs_opts opts = { + .cmds_str = "map_create", + .maps_str = "stack", + }; + + subtest_userns(&opts, userns_map_create); + } + if (test__start_subtest("btf_token")) { + struct bpffs_opts opts = { + .cmds = 1ULL << BPF_BTF_LOAD, + }; + + subtest_userns(&opts, userns_btf_load); + } + if (test__start_subtest("prog_token")) { + struct bpffs_opts opts = { + .cmds_str = "PROG_LOAD", + .progs_str = "XDP", + .attachs_str = "xdp", + }; + + subtest_userns(&opts, userns_prog_load); + } + if (test__start_subtest("obj_priv_map")) { + struct bpffs_opts opts = { + .cmds = bit(BPF_MAP_CREATE), + .maps = bit(BPF_MAP_TYPE_QUEUE), + }; + + subtest_userns(&opts, userns_obj_priv_map); + } + if (test__start_subtest("obj_priv_prog")) { + struct bpffs_opts opts = { + .cmds = bit(BPF_PROG_LOAD), + .progs = bit(BPF_PROG_TYPE_KPROBE), + .attachs = ~0ULL, + }; + + subtest_userns(&opts, userns_obj_priv_prog); + } + if (test__start_subtest("obj_priv_btf_fail")) { + struct bpffs_opts opts = { + /* disallow BTF loading */ + .cmds = bit(BPF_MAP_CREATE) | bit(BPF_PROG_LOAD), + .maps = bit(BPF_MAP_TYPE_STRUCT_OPS), + .progs = bit(BPF_PROG_TYPE_STRUCT_OPS), + .attachs = ~0ULL, + }; + + subtest_userns(&opts, userns_obj_priv_btf_fail); + } + if (test__start_subtest("obj_priv_btf_success")) { + struct bpffs_opts opts = { + /* allow BTF loading */ + .cmds = bit(BPF_BTF_LOAD) | bit(BPF_MAP_CREATE) | bit(BPF_PROG_LOAD), + .maps = bit(BPF_MAP_TYPE_STRUCT_OPS), + .progs = bit(BPF_PROG_TYPE_STRUCT_OPS), + .attachs = ~0ULL, + }; + + subtest_userns(&opts, userns_obj_priv_btf_success); + } + if (test__start_subtest("obj_priv_implicit_token")) { + struct bpffs_opts opts = { + /* allow BTF loading */ + .cmds = bit(BPF_BTF_LOAD) | bit(BPF_MAP_CREATE) | bit(BPF_PROG_LOAD), + .maps = bit(BPF_MAP_TYPE_STRUCT_OPS), + .progs = bit(BPF_PROG_TYPE_STRUCT_OPS), + .attachs = ~0ULL, + }; + + subtest_userns(&opts, userns_obj_priv_implicit_token); + } + if (test__start_subtest("obj_priv_implicit_token_envvar")) { + struct bpffs_opts opts = { + /* allow BTF loading */ + .cmds = bit(BPF_BTF_LOAD) | bit(BPF_MAP_CREATE) | bit(BPF_PROG_LOAD), + .maps = bit(BPF_MAP_TYPE_STRUCT_OPS), + .progs = bit(BPF_PROG_TYPE_STRUCT_OPS), + .attachs = ~0ULL, + }; + + subtest_userns(&opts, userns_obj_priv_implicit_token_envvar); + } +} diff --git a/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c b/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c index ece260cf2c0b66..8269cdee33ae97 100644 --- a/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c +++ b/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c @@ -234,6 +234,177 @@ static void test_attach_api_syms(void) test_attach_api("/proc/self/exe", NULL, &opts); } +static void test_attach_api_fails(void) +{ + LIBBPF_OPTS(bpf_link_create_opts, opts); + const char *path = "/proc/self/exe"; + struct uprobe_multi *skel = NULL; + int prog_fd, link_fd = -1; + unsigned long offset = 0; + + skel = uprobe_multi__open_and_load(); + if (!ASSERT_OK_PTR(skel, "uprobe_multi__open_and_load")) + goto cleanup; + + prog_fd = bpf_program__fd(skel->progs.uprobe_extra); + + /* abnormal cnt */ + opts.uprobe_multi.path = path; + opts.uprobe_multi.offsets = &offset; + opts.uprobe_multi.cnt = INT_MAX; + link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_UPROBE_MULTI, &opts); + if (!ASSERT_ERR(link_fd, "link_fd")) + goto cleanup; + if (!ASSERT_EQ(link_fd, -E2BIG, "big cnt")) + goto cleanup; + + /* cnt is 0 */ + LIBBPF_OPTS_RESET(opts, + .uprobe_multi.path = path, + .uprobe_multi.offsets = (unsigned long *) &offset, + ); + + link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_UPROBE_MULTI, &opts); + if (!ASSERT_ERR(link_fd, "link_fd")) + goto cleanup; + if (!ASSERT_EQ(link_fd, -EINVAL, "cnt_is_zero")) + goto cleanup; + + /* negative offset */ + offset = -1; + opts.uprobe_multi.path = path; + opts.uprobe_multi.offsets = (unsigned long *) &offset; + opts.uprobe_multi.cnt = 1; + + link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_UPROBE_MULTI, &opts); + if (!ASSERT_ERR(link_fd, "link_fd")) + goto cleanup; + if (!ASSERT_EQ(link_fd, -EINVAL, "offset_is_negative")) + goto cleanup; + + /* offsets is NULL */ + LIBBPF_OPTS_RESET(opts, + .uprobe_multi.path = path, + .uprobe_multi.cnt = 1, + ); + + link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_UPROBE_MULTI, &opts); + if (!ASSERT_ERR(link_fd, "link_fd")) + goto cleanup; + if (!ASSERT_EQ(link_fd, -EINVAL, "offsets_is_null")) + goto cleanup; + + /* wrong offsets pointer */ + LIBBPF_OPTS_RESET(opts, + .uprobe_multi.path = path, + .uprobe_multi.offsets = (unsigned long *) 1, + .uprobe_multi.cnt = 1, + ); + + link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_UPROBE_MULTI, &opts); + if (!ASSERT_ERR(link_fd, "link_fd")) + goto cleanup; + if (!ASSERT_EQ(link_fd, -EFAULT, "offsets_is_wrong")) + goto cleanup; + + /* path is NULL */ + offset = 1; + LIBBPF_OPTS_RESET(opts, + .uprobe_multi.offsets = (unsigned long *) &offset, + .uprobe_multi.cnt = 1, + ); + + link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_UPROBE_MULTI, &opts); + if (!ASSERT_ERR(link_fd, "link_fd")) + goto cleanup; + if (!ASSERT_EQ(link_fd, -EINVAL, "path_is_null")) + goto cleanup; + + /* wrong path pointer */ + LIBBPF_OPTS_RESET(opts, + .uprobe_multi.path = (const char *) 1, + .uprobe_multi.offsets = (unsigned long *) &offset, + .uprobe_multi.cnt = 1, + ); + + link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_UPROBE_MULTI, &opts); + if (!ASSERT_ERR(link_fd, "link_fd")) + goto cleanup; + if (!ASSERT_EQ(link_fd, -EFAULT, "path_is_wrong")) + goto cleanup; + + /* wrong path type */ + LIBBPF_OPTS_RESET(opts, + .uprobe_multi.path = "/", + .uprobe_multi.offsets = (unsigned long *) &offset, + .uprobe_multi.cnt = 1, + ); + + link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_UPROBE_MULTI, &opts); + if (!ASSERT_ERR(link_fd, "link_fd")) + goto cleanup; + if (!ASSERT_EQ(link_fd, -EBADF, "path_is_wrong_type")) + goto cleanup; + + /* wrong cookies pointer */ + LIBBPF_OPTS_RESET(opts, + .uprobe_multi.path = path, + .uprobe_multi.offsets = (unsigned long *) &offset, + .uprobe_multi.cookies = (__u64 *) 1ULL, + .uprobe_multi.cnt = 1, + ); + + link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_UPROBE_MULTI, &opts); + if (!ASSERT_ERR(link_fd, "link_fd")) + goto cleanup; + if (!ASSERT_EQ(link_fd, -EFAULT, "cookies_is_wrong")) + goto cleanup; + + /* wrong ref_ctr_offsets pointer */ + LIBBPF_OPTS_RESET(opts, + .uprobe_multi.path = path, + .uprobe_multi.offsets = (unsigned long *) &offset, + .uprobe_multi.cookies = (__u64 *) &offset, + .uprobe_multi.ref_ctr_offsets = (unsigned long *) 1, + .uprobe_multi.cnt = 1, + ); + + link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_UPROBE_MULTI, &opts); + if (!ASSERT_ERR(link_fd, "link_fd")) + goto cleanup; + if (!ASSERT_EQ(link_fd, -EFAULT, "ref_ctr_offsets_is_wrong")) + goto cleanup; + + /* wrong flags */ + LIBBPF_OPTS_RESET(opts, + .uprobe_multi.flags = 1 << 31, + ); + + link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_UPROBE_MULTI, &opts); + if (!ASSERT_ERR(link_fd, "link_fd")) + goto cleanup; + if (!ASSERT_EQ(link_fd, -EINVAL, "wrong_flags")) + goto cleanup; + + /* wrong pid */ + LIBBPF_OPTS_RESET(opts, + .uprobe_multi.path = path, + .uprobe_multi.offsets = (unsigned long *) &offset, + .uprobe_multi.cnt = 1, + .uprobe_multi.pid = -2, + ); + + link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_UPROBE_MULTI, &opts); + if (!ASSERT_ERR(link_fd, "link_fd")) + goto cleanup; + ASSERT_EQ(link_fd, -ESRCH, "pid_is_wrong"); + +cleanup: + if (link_fd >= 0) + close(link_fd); + uprobe_multi__destroy(skel); +} + static void __test_link_api(struct child *child) { int prog_fd, link1_fd = -1, link2_fd = -1, link3_fd = -1, link4_fd = -1; @@ -311,7 +482,7 @@ static void __test_link_api(struct child *child) free(offsets); } -void test_link_api(void) +static void test_link_api(void) { struct child *child; @@ -412,4 +583,6 @@ void test_uprobe_multi_test(void) test_bench_attach_uprobe(); if (test__start_subtest("bench_usdt")) test_bench_attach_usdt(); + if (test__start_subtest("attach_api_fails")) + test_attach_api_fails(); } diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index 8d746642cbd7e6..ac49ec25211d1d 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -6,6 +6,7 @@ #include "verifier_and.skel.h" #include "verifier_array_access.skel.h" #include "verifier_basic_stack.skel.h" +#include "verifier_bitfield_write.skel.h" #include "verifier_bounds.skel.h" #include "verifier_bounds_deduction.skel.h" #include "verifier_bounds_deduction_non_const.skel.h" @@ -116,6 +117,7 @@ static void run_tests_aux(const char *skel_name, void test_verifier_and(void) { RUN(verifier_and); } void test_verifier_basic_stack(void) { RUN(verifier_basic_stack); } +void test_verifier_bitfield_write(void) { RUN(verifier_bitfield_write); } void test_verifier_bounds(void) { RUN(verifier_bounds); } void test_verifier_bounds_deduction(void) { RUN(verifier_bounds_deduction); } void test_verifier_bounds_deduction_non_const(void) { RUN(verifier_bounds_deduction_non_const); } diff --git a/tools/testing/selftests/bpf/prog_tests/verify_pkcs7_sig.c b/tools/testing/selftests/bpf/prog_tests/verify_pkcs7_sig.c index dd7f2bc70048c9..ab0f02faa80cc1 100644 --- a/tools/testing/selftests/bpf/prog_tests/verify_pkcs7_sig.c +++ b/tools/testing/selftests/bpf/prog_tests/verify_pkcs7_sig.c @@ -16,9 +16,12 @@ #include #include #include +#include +#include #include #include "test_verify_pkcs7_sig.skel.h" +#include "test_sig_in_xattr.skel.h" #define MAX_DATA_SIZE (1024 * 1024) #define MAX_SIG_SIZE 1024 @@ -26,6 +29,10 @@ #define VERIFY_USE_SECONDARY_KEYRING (1UL) #define VERIFY_USE_PLATFORM_KEYRING (2UL) +#ifndef SHA256_DIGEST_SIZE +#define SHA256_DIGEST_SIZE 32 +#endif + /* In stripped ARM and x86-64 modules, ~ is surprisingly rare. */ #define MODULE_SIG_STRING "~Module signature appended~\n" @@ -254,7 +261,7 @@ static int populate_data_item_mod(struct data *data_item) return ret; } -void test_verify_pkcs7_sig(void) +static void test_verify_pkcs7_sig_from_map(void) { libbpf_print_fn_t old_print_cb; char tmp_dir_template[] = "/tmp/verify_sigXXXXXX"; @@ -400,3 +407,159 @@ void test_verify_pkcs7_sig(void) skel->bss->monitored_pid = 0; test_verify_pkcs7_sig__destroy(skel); } + +static int get_signature_size(const char *sig_path) +{ + struct stat st; + + if (stat(sig_path, &st) == -1) + return -1; + + return st.st_size; +} + +static int add_signature_to_xattr(const char *data_path, const char *sig_path) +{ + char sig[MAX_SIG_SIZE] = {0}; + int fd, size, ret; + + if (sig_path) { + fd = open(sig_path, O_RDONLY); + if (fd < 0) + return -1; + + size = read(fd, sig, MAX_SIG_SIZE); + close(fd); + if (size <= 0) + return -1; + } else { + /* no sig_path, just write 32 bytes of zeros */ + size = 32; + } + ret = setxattr(data_path, "user.sig", sig, size, 0); + if (!ASSERT_OK(ret, "setxattr")) + return -1; + + return 0; +} + +static int test_open_file(struct test_sig_in_xattr *skel, char *data_path, + pid_t pid, bool should_success, char *name) +{ + int ret; + + skel->bss->monitored_pid = pid; + ret = open(data_path, O_RDONLY); + close(ret); + skel->bss->monitored_pid = 0; + + if (should_success) { + if (!ASSERT_GE(ret, 0, name)) + return -1; + } else { + if (!ASSERT_LT(ret, 0, name)) + return -1; + } + return 0; +} + +static void test_pkcs7_sig_fsverity(void) +{ + char data_path[PATH_MAX]; + char sig_path[PATH_MAX]; + char tmp_dir_template[] = "/tmp/verify_sigXXXXXX"; + char *tmp_dir; + struct test_sig_in_xattr *skel = NULL; + pid_t pid; + int ret; + + tmp_dir = mkdtemp(tmp_dir_template); + if (!ASSERT_OK_PTR(tmp_dir, "mkdtemp")) + return; + + snprintf(data_path, PATH_MAX, "%s/data-file", tmp_dir); + snprintf(sig_path, PATH_MAX, "%s/sig-file", tmp_dir); + + ret = _run_setup_process(tmp_dir, "setup"); + if (!ASSERT_OK(ret, "_run_setup_process")) + goto out; + + ret = _run_setup_process(tmp_dir, "fsverity-create-sign"); + + if (ret) { + printf("%s: SKIP: fsverity [sign|enable] doesn't work.\n" + "To run this test, try enable CONFIG_FS_VERITY and enable FSVerity for the filesystem.\n", + __func__); + test__skip(); + goto out; + } + + skel = test_sig_in_xattr__open(); + if (!ASSERT_OK_PTR(skel, "test_sig_in_xattr__open")) + goto out; + ret = get_signature_size(sig_path); + if (!ASSERT_GT(ret, 0, "get_signature_size")) + goto out; + skel->bss->sig_size = ret; + skel->bss->user_keyring_serial = syscall(__NR_request_key, "keyring", + "ebpf_testing_keyring", NULL, + KEY_SPEC_SESSION_KEYRING); + memcpy(skel->bss->digest, "FSVerity", 8); + + ret = test_sig_in_xattr__load(skel); + if (!ASSERT_OK(ret, "test_sig_in_xattr__load")) + goto out; + + ret = test_sig_in_xattr__attach(skel); + if (!ASSERT_OK(ret, "test_sig_in_xattr__attach")) + goto out; + + pid = getpid(); + + /* Case 1: fsverity is not enabled, open should succeed */ + if (test_open_file(skel, data_path, pid, true, "open_1")) + goto out; + + /* Case 2: fsverity is enabled, xattr is missing, open should + * fail + */ + ret = _run_setup_process(tmp_dir, "fsverity-enable"); + if (!ASSERT_OK(ret, "fsverity-enable")) + goto out; + if (test_open_file(skel, data_path, pid, false, "open_2")) + goto out; + + /* Case 3: fsverity is enabled, xattr has valid signature, open + * should succeed + */ + ret = add_signature_to_xattr(data_path, sig_path); + if (!ASSERT_OK(ret, "add_signature_to_xattr_1")) + goto out; + + if (test_open_file(skel, data_path, pid, true, "open_3")) + goto out; + + /* Case 4: fsverity is enabled, xattr has invalid signature, open + * should fail + */ + ret = add_signature_to_xattr(data_path, NULL); + if (!ASSERT_OK(ret, "add_signature_to_xattr_2")) + goto out; + test_open_file(skel, data_path, pid, false, "open_4"); + +out: + _run_setup_process(tmp_dir, "cleanup"); + if (!skel) + return; + + skel->bss->monitored_pid = 0; + test_sig_in_xattr__destroy(skel); +} + +void test_verify_pkcs7_sig(void) +{ + if (test__start_subtest("pkcs7_sig_from_map")) + test_verify_pkcs7_sig_from_map(); + if (test__start_subtest("pkcs7_sig_fsverity")) + test_pkcs7_sig_fsverity(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_context_test_run.c b/tools/testing/selftests/bpf/prog_tests/xdp_context_test_run.c index ab4952b9fb1d4a..e6a783c7f5db9c 100644 --- a/tools/testing/selftests/bpf/prog_tests/xdp_context_test_run.c +++ b/tools/testing/selftests/bpf/prog_tests/xdp_context_test_run.c @@ -77,8 +77,8 @@ void test_xdp_context_test_run(void) test_xdp_context_error(prog_fd, opts, 4, sizeof(__u32), sizeof(data), 0, 0, 0); - /* Meta data must be 32 bytes or smaller */ - test_xdp_context_error(prog_fd, opts, 0, 36, sizeof(data), 0, 0, 0); + /* Meta data must be 255 bytes or smaller */ + test_xdp_context_error(prog_fd, opts, 0, 256, sizeof(data), 0, 0, 0); /* Total size of data must match data_end - data_meta */ test_xdp_context_error(prog_fd, opts, 0, sizeof(__u32), diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_metadata.c b/tools/testing/selftests/bpf/prog_tests/xdp_metadata.c index 33cdf88efa6b92..05edcf32f52898 100644 --- a/tools/testing/selftests/bpf/prog_tests/xdp_metadata.c +++ b/tools/testing/selftests/bpf/prog_tests/xdp_metadata.c @@ -20,7 +20,7 @@ #define UDP_PAYLOAD_BYTES 4 -#define AF_XDP_SOURCE_PORT 1234 +#define UDP_SOURCE_PORT 1234 #define AF_XDP_CONSUMER_PORT 8080 #define UMEM_NUM 16 @@ -33,6 +33,18 @@ #define RX_ADDR "10.0.0.2" #define PREFIX_LEN "8" #define FAMILY AF_INET +#define TX_NETNS_NAME "xdp_metadata_tx" +#define RX_NETNS_NAME "xdp_metadata_rx" +#define TX_MAC "00:00:00:00:00:01" +#define RX_MAC "00:00:00:00:00:02" + +#define VLAN_ID 59 +#define VLAN_PROTO "802.1Q" +#define VLAN_PID htons(ETH_P_8021Q) +#define TX_NAME_VLAN TX_NAME "." TO_STR(VLAN_ID) + +#define XDP_RSS_TYPE_L4 BIT(3) +#define VLAN_VID_MASK 0xfff struct xsk { void *umem_area; @@ -181,7 +193,7 @@ static int generate_packet(struct xsk *xsk, __u16 dst_port) ASSERT_EQ(inet_pton(FAMILY, RX_ADDR, &iph->daddr), 1, "inet_pton(RX_ADDR)"); ip_csum(iph); - udph->source = htons(AF_XDP_SOURCE_PORT); + udph->source = htons(UDP_SOURCE_PORT); udph->dest = htons(dst_port); udph->len = htons(sizeof(*udph) + UDP_PAYLOAD_BYTES); udph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, @@ -204,6 +216,30 @@ static int generate_packet(struct xsk *xsk, __u16 dst_port) return 0; } +static int generate_packet_inet(void) +{ + char udp_payload[UDP_PAYLOAD_BYTES]; + struct sockaddr_in rx_addr; + int sock_fd, err = 0; + + /* Build a packet */ + memset(udp_payload, 0xAA, UDP_PAYLOAD_BYTES); + rx_addr.sin_addr.s_addr = inet_addr(RX_ADDR); + rx_addr.sin_family = AF_INET; + rx_addr.sin_port = htons(AF_XDP_CONSUMER_PORT); + + sock_fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); + if (!ASSERT_GE(sock_fd, 0, "socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)")) + return sock_fd; + + err = sendto(sock_fd, udp_payload, UDP_PAYLOAD_BYTES, MSG_DONTWAIT, + (void *)&rx_addr, sizeof(rx_addr)); + ASSERT_GE(err, 0, "sendto"); + + close(sock_fd); + return err; +} + static void complete_tx(struct xsk *xsk) { struct xsk_tx_metadata *meta; @@ -236,7 +272,7 @@ static void refill_rx(struct xsk *xsk, __u64 addr) } } -static int verify_xsk_metadata(struct xsk *xsk) +static int verify_xsk_metadata(struct xsk *xsk, bool sent_from_af_xdp) { const struct xdp_desc *rx_desc; struct pollfd fds = {}; @@ -290,17 +326,42 @@ static int verify_xsk_metadata(struct xsk *xsk) if (!ASSERT_NEQ(meta->rx_hash, 0, "rx_hash")) return -1; + if (!sent_from_af_xdp) { + if (!ASSERT_NEQ(meta->rx_hash_type & XDP_RSS_TYPE_L4, 0, "rx_hash_type")) + return -1; + + if (!ASSERT_EQ(meta->rx_vlan_tci & VLAN_VID_MASK, VLAN_ID, "rx_vlan_tci")) + return -1; + + if (!ASSERT_EQ(meta->rx_vlan_proto, VLAN_PID, "rx_vlan_proto")) + return -1; + goto done; + } + ASSERT_EQ(meta->rx_hash_type, 0, "rx_hash_type"); /* checksum offload */ ASSERT_EQ(udph->check, htons(0x721c), "csum"); +done: xsk_ring_cons__release(&xsk->rx, 1); refill_rx(xsk, comp_addr); return 0; } +static void switch_ns_to_rx(struct nstoken **tok) +{ + close_netns(*tok); + *tok = open_netns(RX_NETNS_NAME); +} + +static void switch_ns_to_tx(struct nstoken **tok) +{ + close_netns(*tok); + *tok = open_netns(TX_NETNS_NAME); +} + void test_xdp_metadata(void) { struct xdp_metadata2 *bpf_obj2 = NULL; @@ -318,27 +379,35 @@ void test_xdp_metadata(void) int sock_fd; int ret; - /* Setup new networking namespace, with a veth pair. */ + /* Setup new networking namespaces, with a veth pair. */ + SYS(out, "ip netns add " TX_NETNS_NAME); + SYS(out, "ip netns add " RX_NETNS_NAME); - SYS(out, "ip netns add xdp_metadata"); - tok = open_netns("xdp_metadata"); + tok = open_netns(TX_NETNS_NAME); SYS(out, "ip link add numtxqueues 1 numrxqueues 1 " TX_NAME " type veth peer " RX_NAME " numtxqueues 1 numrxqueues 1"); - SYS(out, "ip link set dev " TX_NAME " address 00:00:00:00:00:01"); - SYS(out, "ip link set dev " RX_NAME " address 00:00:00:00:00:02"); + SYS(out, "ip link set " RX_NAME " netns " RX_NETNS_NAME); + + SYS(out, "ip link set dev " TX_NAME " address " TX_MAC); SYS(out, "ip link set dev " TX_NAME " up"); + + SYS(out, "ip link add link " TX_NAME " " TX_NAME_VLAN + " type vlan proto " VLAN_PROTO " id " TO_STR(VLAN_ID)); + SYS(out, "ip link set dev " TX_NAME_VLAN " up"); + SYS(out, "ip addr add " TX_ADDR "/" PREFIX_LEN " dev " TX_NAME_VLAN); + + /* Avoid ARP calls */ + SYS(out, "ip -4 neigh add " RX_ADDR " lladdr " RX_MAC " dev " TX_NAME_VLAN); + + switch_ns_to_rx(&tok); + + SYS(out, "ip link set dev " RX_NAME " address " RX_MAC); SYS(out, "ip link set dev " RX_NAME " up"); - SYS(out, "ip addr add " TX_ADDR "/" PREFIX_LEN " dev " TX_NAME); SYS(out, "ip addr add " RX_ADDR "/" PREFIX_LEN " dev " RX_NAME); rx_ifindex = if_nametoindex(RX_NAME); - tx_ifindex = if_nametoindex(TX_NAME); - - /* Setup separate AF_XDP for TX and RX interfaces. */ - ret = open_xsk(tx_ifindex, &tx_xsk); - if (!ASSERT_OK(ret, "open_xsk(TX_NAME)")) - goto out; + /* Setup separate AF_XDP for RX interface. */ ret = open_xsk(rx_ifindex, &rx_xsk); if (!ASSERT_OK(ret, "open_xsk(RX_NAME)")) @@ -379,18 +448,38 @@ void test_xdp_metadata(void) if (!ASSERT_GE(ret, 0, "bpf_map_update_elem")) goto out; - /* Send packet destined to RX AF_XDP socket. */ + switch_ns_to_tx(&tok); + + /* Setup separate AF_XDP for TX interface nad send packet to the RX socket. */ + tx_ifindex = if_nametoindex(TX_NAME); + ret = open_xsk(tx_ifindex, &tx_xsk); + if (!ASSERT_OK(ret, "open_xsk(TX_NAME)")) + goto out; + if (!ASSERT_GE(generate_packet(&tx_xsk, AF_XDP_CONSUMER_PORT), 0, "generate AF_XDP_CONSUMER_PORT")) goto out; - /* Verify AF_XDP RX packet has proper metadata. */ - if (!ASSERT_GE(verify_xsk_metadata(&rx_xsk), 0, + switch_ns_to_rx(&tok); + + /* Verify packet sent from AF_XDP has proper metadata. */ + if (!ASSERT_GE(verify_xsk_metadata(&rx_xsk, true), 0, "verify_xsk_metadata")) goto out; + switch_ns_to_tx(&tok); complete_tx(&tx_xsk); + /* Now check metadata of packet, generated with network stack */ + if (!ASSERT_GE(generate_packet_inet(), 0, "generate UDP packet")) + goto out; + + switch_ns_to_rx(&tok); + + if (!ASSERT_GE(verify_xsk_metadata(&rx_xsk, false), 0, + "verify_xsk_metadata")) + goto out; + /* Make sure freplace correctly picks up original bound device * and doesn't crash. */ @@ -408,11 +497,15 @@ void test_xdp_metadata(void) if (!ASSERT_OK(xdp_metadata2__attach(bpf_obj2), "attach freplace")) goto out; + switch_ns_to_tx(&tok); + /* Send packet to trigger . */ if (!ASSERT_GE(generate_packet(&tx_xsk, AF_XDP_CONSUMER_PORT), 0, "generate freplace packet")) goto out; + switch_ns_to_rx(&tok); + while (!retries--) { if (bpf_obj2->bss->called) break; @@ -427,5 +520,6 @@ void test_xdp_metadata(void) xdp_metadata__destroy(bpf_obj); if (tok) close_netns(tok); - SYS_NOFAIL("ip netns del xdp_metadata"); + SYS_NOFAIL("ip netns del " RX_NETNS_NAME); + SYS_NOFAIL("ip netns del " TX_NETNS_NAME); } diff --git a/tools/testing/selftests/bpf/progs/access_map_in_map.c b/tools/testing/selftests/bpf/progs/access_map_in_map.c new file mode 100644 index 00000000000000..1126871c2ebd83 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/access_map_in_map.c @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2023. Huawei Technologies Co., Ltd */ +#include +#include +#include + +#include "bpf_misc.h" + +struct inner_map_type { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(key_size, 4); + __uint(value_size, 4); + __uint(max_entries, 1); +} inner_map SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); + __type(key, int); + __type(value, int); + __uint(max_entries, 1); + __array(values, struct inner_map_type); +} outer_array_map SEC(".maps") = { + .values = { + [0] = &inner_map, + }, +}; + +struct { + __uint(type, BPF_MAP_TYPE_HASH_OF_MAPS); + __type(key, int); + __type(value, int); + __uint(max_entries, 1); + __array(values, struct inner_map_type); +} outer_htab_map SEC(".maps") = { + .values = { + [0] = &inner_map, + }, +}; + +char _license[] SEC("license") = "GPL"; + +int tgid = 0; + +static int acc_map_in_map(void *outer_map) +{ + int i, key, value = 0xdeadbeef; + void *inner_map; + + if ((bpf_get_current_pid_tgid() >> 32) != tgid) + return 0; + + /* Find nonexistent inner map */ + key = 1; + inner_map = bpf_map_lookup_elem(outer_map, &key); + if (inner_map) + return 0; + + /* Find the old inner map */ + key = 0; + inner_map = bpf_map_lookup_elem(outer_map, &key); + if (!inner_map) + return 0; + + /* Wait for the old inner map to be replaced */ + for (i = 0; i < 2048; i++) + bpf_map_update_elem(inner_map, &key, &value, 0); + + return 0; +} + +SEC("?kprobe/" SYS_PREFIX "sys_getpgid") +int access_map_in_array(void *ctx) +{ + return acc_map_in_map(&outer_array_map); +} + +SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") +int sleepable_access_map_in_array(void *ctx) +{ + return acc_map_in_map(&outer_array_map); +} + +SEC("?kprobe/" SYS_PREFIX "sys_getpgid") +int access_map_in_htab(void *ctx) +{ + return acc_map_in_map(&outer_htab_map); +} + +SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") +int sleepable_access_map_in_htab(void *ctx) +{ + return acc_map_in_map(&outer_htab_map); +} diff --git a/tools/testing/selftests/bpf/progs/bpf_misc.h b/tools/testing/selftests/bpf/progs/bpf_misc.h index 799fff4995d870..2fd59970c43a2d 100644 --- a/tools/testing/selftests/bpf/progs/bpf_misc.h +++ b/tools/testing/selftests/bpf/progs/bpf_misc.h @@ -71,6 +71,7 @@ #define __retval_unpriv(val) __attribute__((btf_decl_tag("comment:test_retval_unpriv="#val))) #define __auxiliary __attribute__((btf_decl_tag("comment:test_auxiliary"))) #define __auxiliary_unpriv __attribute__((btf_decl_tag("comment:test_auxiliary_unpriv"))) +#define __btf_path(path) __attribute__((btf_decl_tag("comment:test_btf_path=" path))) /* Convenience macro for use with 'asm volatile' blocks */ #define __naked __attribute__((naked)) diff --git a/tools/testing/selftests/bpf/progs/bpf_tracing_net.h b/tools/testing/selftests/bpf/progs/bpf_tracing_net.h index 0b793a102791f7..1bdc680b0e0e26 100644 --- a/tools/testing/selftests/bpf/progs/bpf_tracing_net.h +++ b/tools/testing/selftests/bpf/progs/bpf_tracing_net.h @@ -26,6 +26,7 @@ #define IPV6_AUTOFLOWLABEL 70 #define TC_ACT_UNSPEC (-1) +#define TC_ACT_OK 0 #define TC_ACT_SHOT 2 #define SOL_TCP 6 diff --git a/tools/testing/selftests/bpf/progs/cgrp_ls_recursion.c b/tools/testing/selftests/bpf/progs/cgrp_ls_recursion.c index a043d8fefdac12..610c2427fd932a 100644 --- a/tools/testing/selftests/bpf/progs/cgrp_ls_recursion.c +++ b/tools/testing/selftests/bpf/progs/cgrp_ls_recursion.c @@ -21,50 +21,100 @@ struct { __type(value, long); } map_b SEC(".maps"); +int target_hid = 0; +bool is_cgroup1 = 0; + +struct cgroup *bpf_task_get_cgroup1(struct task_struct *task, int hierarchy_id) __ksym; +void bpf_cgroup_release(struct cgroup *cgrp) __ksym; + +static void __on_lookup(struct cgroup *cgrp) +{ + bpf_cgrp_storage_delete(&map_a, cgrp); + bpf_cgrp_storage_delete(&map_b, cgrp); +} + SEC("fentry/bpf_local_storage_lookup") int BPF_PROG(on_lookup) { struct task_struct *task = bpf_get_current_task_btf(); + struct cgroup *cgrp; + + if (is_cgroup1) { + cgrp = bpf_task_get_cgroup1(task, target_hid); + if (!cgrp) + return 0; - bpf_cgrp_storage_delete(&map_a, task->cgroups->dfl_cgrp); - bpf_cgrp_storage_delete(&map_b, task->cgroups->dfl_cgrp); + __on_lookup(cgrp); + bpf_cgroup_release(cgrp); + return 0; + } + + __on_lookup(task->cgroups->dfl_cgrp); return 0; } -SEC("fentry/bpf_local_storage_update") -int BPF_PROG(on_update) +static void __on_update(struct cgroup *cgrp) { - struct task_struct *task = bpf_get_current_task_btf(); long *ptr; - ptr = bpf_cgrp_storage_get(&map_a, task->cgroups->dfl_cgrp, 0, - BPF_LOCAL_STORAGE_GET_F_CREATE); + ptr = bpf_cgrp_storage_get(&map_a, cgrp, 0, BPF_LOCAL_STORAGE_GET_F_CREATE); if (ptr) *ptr += 1; - ptr = bpf_cgrp_storage_get(&map_b, task->cgroups->dfl_cgrp, 0, - BPF_LOCAL_STORAGE_GET_F_CREATE); + ptr = bpf_cgrp_storage_get(&map_b, cgrp, 0, BPF_LOCAL_STORAGE_GET_F_CREATE); if (ptr) *ptr += 1; +} +SEC("fentry/bpf_local_storage_update") +int BPF_PROG(on_update) +{ + struct task_struct *task = bpf_get_current_task_btf(); + struct cgroup *cgrp; + + if (is_cgroup1) { + cgrp = bpf_task_get_cgroup1(task, target_hid); + if (!cgrp) + return 0; + + __on_update(cgrp); + bpf_cgroup_release(cgrp); + return 0; + } + + __on_update(task->cgroups->dfl_cgrp); return 0; } -SEC("tp_btf/sys_enter") -int BPF_PROG(on_enter, struct pt_regs *regs, long id) +static void __on_enter(struct pt_regs *regs, long id, struct cgroup *cgrp) { - struct task_struct *task; long *ptr; - task = bpf_get_current_task_btf(); - ptr = bpf_cgrp_storage_get(&map_a, task->cgroups->dfl_cgrp, 0, - BPF_LOCAL_STORAGE_GET_F_CREATE); + ptr = bpf_cgrp_storage_get(&map_a, cgrp, 0, BPF_LOCAL_STORAGE_GET_F_CREATE); if (ptr) *ptr = 200; - ptr = bpf_cgrp_storage_get(&map_b, task->cgroups->dfl_cgrp, 0, - BPF_LOCAL_STORAGE_GET_F_CREATE); + ptr = bpf_cgrp_storage_get(&map_b, cgrp, 0, BPF_LOCAL_STORAGE_GET_F_CREATE); if (ptr) *ptr = 100; +} + +SEC("tp_btf/sys_enter") +int BPF_PROG(on_enter, struct pt_regs *regs, long id) +{ + struct task_struct *task = bpf_get_current_task_btf(); + struct cgroup *cgrp; + + if (is_cgroup1) { + cgrp = bpf_task_get_cgroup1(task, target_hid); + if (!cgrp) + return 0; + + __on_enter(regs, id, cgrp); + bpf_cgroup_release(cgrp); + return 0; + } + + __on_enter(regs, id, task->cgroups->dfl_cgrp); return 0; } diff --git a/tools/testing/selftests/bpf/progs/cgrp_ls_sleepable.c b/tools/testing/selftests/bpf/progs/cgrp_ls_sleepable.c index 4c7844e1dbfa58..facedd8b825007 100644 --- a/tools/testing/selftests/bpf/progs/cgrp_ls_sleepable.c +++ b/tools/testing/selftests/bpf/progs/cgrp_ls_sleepable.c @@ -17,7 +17,11 @@ struct { __u32 target_pid; __u64 cgroup_id; +int target_hid; +bool is_cgroup1; +struct cgroup *bpf_task_get_cgroup1(struct task_struct *task, int hierarchy_id) __ksym; +void bpf_cgroup_release(struct cgroup *cgrp) __ksym; void bpf_rcu_read_lock(void) __ksym; void bpf_rcu_read_unlock(void) __ksym; @@ -37,23 +41,50 @@ int cgroup_iter(struct bpf_iter__cgroup *ctx) return 0; } +static void __no_rcu_lock(struct cgroup *cgrp) +{ + long *ptr; + + /* Note that trace rcu is held in sleepable prog, so we can use + * bpf_cgrp_storage_get() in sleepable prog. + */ + ptr = bpf_cgrp_storage_get(&map_a, cgrp, 0, + BPF_LOCAL_STORAGE_GET_F_CREATE); + if (ptr) + cgroup_id = cgrp->kn->id; +} + SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") -int no_rcu_lock(void *ctx) +int cgrp1_no_rcu_lock(void *ctx) { struct task_struct *task; struct cgroup *cgrp; - long *ptr; + + task = bpf_get_current_task_btf(); + if (task->pid != target_pid) + return 0; + + /* bpf_task_get_cgroup1 can work in sleepable prog */ + cgrp = bpf_task_get_cgroup1(task, target_hid); + if (!cgrp) + return 0; + + __no_rcu_lock(cgrp); + bpf_cgroup_release(cgrp); + return 0; +} + +SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") +int no_rcu_lock(void *ctx) +{ + struct task_struct *task; task = bpf_get_current_task_btf(); if (task->pid != target_pid) return 0; /* task->cgroups is untrusted in sleepable prog outside of RCU CS */ - cgrp = task->cgroups->dfl_cgrp; - ptr = bpf_cgrp_storage_get(&map_a, cgrp, 0, - BPF_LOCAL_STORAGE_GET_F_CREATE); - if (ptr) - cgroup_id = cgrp->kn->id; + __no_rcu_lock(task->cgroups->dfl_cgrp); return 0; } @@ -68,6 +99,22 @@ int yes_rcu_lock(void *ctx) if (task->pid != target_pid) return 0; + if (is_cgroup1) { + bpf_rcu_read_lock(); + cgrp = bpf_task_get_cgroup1(task, target_hid); + if (!cgrp) { + bpf_rcu_read_unlock(); + return 0; + } + + ptr = bpf_cgrp_storage_get(&map_a, cgrp, 0, BPF_LOCAL_STORAGE_GET_F_CREATE); + if (ptr) + cgroup_id = cgrp->kn->id; + bpf_cgroup_release(cgrp); + bpf_rcu_read_unlock(); + return 0; + } + bpf_rcu_read_lock(); cgrp = task->cgroups->dfl_cgrp; /* cgrp is trusted under RCU CS */ diff --git a/tools/testing/selftests/bpf/progs/cgrp_ls_tp_btf.c b/tools/testing/selftests/bpf/progs/cgrp_ls_tp_btf.c index 9ebb8e2fe541c1..1c348f000f38d1 100644 --- a/tools/testing/selftests/bpf/progs/cgrp_ls_tp_btf.c +++ b/tools/testing/selftests/bpf/progs/cgrp_ls_tp_btf.c @@ -27,62 +27,100 @@ pid_t target_pid = 0; int mismatch_cnt = 0; int enter_cnt = 0; int exit_cnt = 0; +int target_hid = 0; +bool is_cgroup1 = 0; -SEC("tp_btf/sys_enter") -int BPF_PROG(on_enter, struct pt_regs *regs, long id) +struct cgroup *bpf_task_get_cgroup1(struct task_struct *task, int hierarchy_id) __ksym; +void bpf_cgroup_release(struct cgroup *cgrp) __ksym; + +static void __on_enter(struct pt_regs *regs, long id, struct cgroup *cgrp) { - struct task_struct *task; long *ptr; int err; - task = bpf_get_current_task_btf(); - if (task->pid != target_pid) - return 0; - /* populate value 0 */ - ptr = bpf_cgrp_storage_get(&map_a, task->cgroups->dfl_cgrp, 0, + ptr = bpf_cgrp_storage_get(&map_a, cgrp, 0, BPF_LOCAL_STORAGE_GET_F_CREATE); if (!ptr) - return 0; + return; /* delete value 0 */ - err = bpf_cgrp_storage_delete(&map_a, task->cgroups->dfl_cgrp); + err = bpf_cgrp_storage_delete(&map_a, cgrp); if (err) - return 0; + return; /* value is not available */ - ptr = bpf_cgrp_storage_get(&map_a, task->cgroups->dfl_cgrp, 0, 0); + ptr = bpf_cgrp_storage_get(&map_a, cgrp, 0, 0); if (ptr) - return 0; + return; /* re-populate the value */ - ptr = bpf_cgrp_storage_get(&map_a, task->cgroups->dfl_cgrp, 0, + ptr = bpf_cgrp_storage_get(&map_a, cgrp, 0, BPF_LOCAL_STORAGE_GET_F_CREATE); if (!ptr) - return 0; + return; __sync_fetch_and_add(&enter_cnt, 1); *ptr = MAGIC_VALUE + enter_cnt; - - return 0; } -SEC("tp_btf/sys_exit") -int BPF_PROG(on_exit, struct pt_regs *regs, long id) +SEC("tp_btf/sys_enter") +int BPF_PROG(on_enter, struct pt_regs *regs, long id) { struct task_struct *task; - long *ptr; + struct cgroup *cgrp; task = bpf_get_current_task_btf(); if (task->pid != target_pid) return 0; - ptr = bpf_cgrp_storage_get(&map_a, task->cgroups->dfl_cgrp, 0, + if (is_cgroup1) { + cgrp = bpf_task_get_cgroup1(task, target_hid); + if (!cgrp) + return 0; + + __on_enter(regs, id, cgrp); + bpf_cgroup_release(cgrp); + return 0; + } + + __on_enter(regs, id, task->cgroups->dfl_cgrp); + return 0; +} + +static void __on_exit(struct pt_regs *regs, long id, struct cgroup *cgrp) +{ + long *ptr; + + ptr = bpf_cgrp_storage_get(&map_a, cgrp, 0, BPF_LOCAL_STORAGE_GET_F_CREATE); if (!ptr) - return 0; + return; __sync_fetch_and_add(&exit_cnt, 1); if (*ptr != MAGIC_VALUE + exit_cnt) __sync_fetch_and_add(&mismatch_cnt, 1); +} + +SEC("tp_btf/sys_exit") +int BPF_PROG(on_exit, struct pt_regs *regs, long id) +{ + struct task_struct *task; + struct cgroup *cgrp; + + task = bpf_get_current_task_btf(); + if (task->pid != target_pid) + return 0; + + if (is_cgroup1) { + cgrp = bpf_task_get_cgroup1(task, target_hid); + if (!cgrp) + return 0; + + __on_exit(regs, id, cgrp); + bpf_cgroup_release(cgrp); + return 0; + } + + __on_exit(regs, id, task->cgroups->dfl_cgrp); return 0; } diff --git a/tools/testing/selftests/bpf/progs/cpumask_common.h b/tools/testing/selftests/bpf/progs/cpumask_common.h index b15c588ace15df..0cd4aebb97cf59 100644 --- a/tools/testing/selftests/bpf/progs/cpumask_common.h +++ b/tools/testing/selftests/bpf/progs/cpumask_common.h @@ -54,6 +54,7 @@ bool bpf_cpumask_full(const struct cpumask *cpumask) __ksym; void bpf_cpumask_copy(struct bpf_cpumask *dst, const struct cpumask *src) __ksym; u32 bpf_cpumask_any_distribute(const struct cpumask *src) __ksym; u32 bpf_cpumask_any_and_distribute(const struct cpumask *src1, const struct cpumask *src2) __ksym; +u32 bpf_cpumask_weight(const struct cpumask *cpumask) __ksym; void bpf_rcu_read_lock(void) __ksym; void bpf_rcu_read_unlock(void) __ksym; diff --git a/tools/testing/selftests/bpf/progs/cpumask_success.c b/tools/testing/selftests/bpf/progs/cpumask_success.c index 674a63424dee3e..fc3666edf4561a 100644 --- a/tools/testing/selftests/bpf/progs/cpumask_success.c +++ b/tools/testing/selftests/bpf/progs/cpumask_success.c @@ -460,6 +460,49 @@ int BPF_PROG(test_global_mask_rcu, struct task_struct *task, u64 clone_flags) return 0; } +SEC("tp_btf/task_newtask") +int BPF_PROG(test_cpumask_weight, struct task_struct *task, u64 clone_flags) +{ + struct bpf_cpumask *local; + + if (!is_test_task()) + return 0; + + local = create_cpumask(); + if (!local) + return 0; + + if (bpf_cpumask_weight(cast(local)) != 0) { + err = 3; + goto out; + } + + bpf_cpumask_set_cpu(0, local); + if (bpf_cpumask_weight(cast(local)) != 1) { + err = 4; + goto out; + } + + /* + * Make sure that adding additional CPUs changes the weight. Test to + * see whether the CPU was set to account for running on UP machines. + */ + bpf_cpumask_set_cpu(1, local); + if (bpf_cpumask_test_cpu(1, cast(local)) && bpf_cpumask_weight(cast(local)) != 2) { + err = 5; + goto out; + } + + bpf_cpumask_clear(local); + if (bpf_cpumask_weight(cast(local)) != 0) { + err = 6; + goto out; + } +out: + bpf_cpumask_release(local); + return 0; +} + SEC("tp_btf/task_newtask") __success int BPF_PROG(test_refcount_null_tracking, struct task_struct *task, u64 clone_flags) diff --git a/tools/testing/selftests/bpf/progs/exceptions_assert.c b/tools/testing/selftests/bpf/progs/exceptions_assert.c index 49efaed143fc2b..0ef81040da5960 100644 --- a/tools/testing/selftests/bpf/progs/exceptions_assert.c +++ b/tools/testing/selftests/bpf/progs/exceptions_assert.c @@ -125,7 +125,7 @@ int check_assert_generic(struct __sk_buff *ctx) } SEC("?fentry/bpf_check") -__failure __msg("At program exit the register R0 has value (0x40; 0x0)") +__failure __msg("At program exit the register R1 has smin=64 smax=64") int check_assert_with_return(void *ctx) { bpf_assert_with(!ctx, 64); diff --git a/tools/testing/selftests/bpf/progs/exceptions_fail.c b/tools/testing/selftests/bpf/progs/exceptions_fail.c index 8c0ef2742208ae..9cceb652114335 100644 --- a/tools/testing/selftests/bpf/progs/exceptions_fail.c +++ b/tools/testing/selftests/bpf/progs/exceptions_fail.c @@ -308,7 +308,7 @@ int reject_set_exception_cb_bad_ret1(void *ctx) } SEC("?fentry/bpf_check") -__failure __msg("At program exit the register R0 has value (0x40; 0x0) should") +__failure __msg("At program exit the register R1 has smin=64 smax=64 should") int reject_set_exception_cb_bad_ret2(void *ctx) { bpf_throw(64); diff --git a/tools/testing/selftests/bpf/progs/freplace_dead_global_func.c b/tools/testing/selftests/bpf/progs/freplace_dead_global_func.c new file mode 100644 index 00000000000000..e6a75f86cac64e --- /dev/null +++ b/tools/testing/selftests/bpf/progs/freplace_dead_global_func.c @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include + +SEC("freplace") +int freplace_prog(void) +{ + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/iters.c b/tools/testing/selftests/bpf/progs/iters.c index b2181f850d3e55..3aca3dc145b552 100644 --- a/tools/testing/selftests/bpf/progs/iters.c +++ b/tools/testing/selftests/bpf/progs/iters.c @@ -846,7 +846,7 @@ __naked int delayed_precision_mark(void) "call %[bpf_iter_num_next];" "if r0 == 0 goto 2f;" "if r6 != 42 goto 3f;" - "r7 = -32;" + "r7 = -33;" "call %[bpf_get_prandom_u32];" "r6 = r0;" "goto 1b;\n" diff --git a/tools/testing/selftests/bpf/progs/local_kptr_stash.c b/tools/testing/selftests/bpf/progs/local_kptr_stash.c index 1769fdff6aeae6..75043ffc5dad0b 100644 --- a/tools/testing/selftests/bpf/progs/local_kptr_stash.c +++ b/tools/testing/selftests/bpf/progs/local_kptr_stash.c @@ -37,11 +37,18 @@ struct plain_local { long data; }; +struct local_with_root { + long key; + struct bpf_spin_lock l; + struct bpf_rb_root r __contains(node_data, node); +}; + struct map_value { struct prog_test_ref_kfunc *not_kptr; struct prog_test_ref_kfunc __kptr *val; struct node_data __kptr *node; struct plain_local __kptr *plain; + struct local_with_root __kptr *local_root; }; /* This is necessary so that LLVM generates BTF for node_data struct @@ -65,6 +72,17 @@ struct { __uint(max_entries, 2); } some_nodes SEC(".maps"); +static bool less(struct bpf_rb_node *a, const struct bpf_rb_node *b) +{ + struct node_data *node_a; + struct node_data *node_b; + + node_a = container_of(a, struct node_data, node); + node_b = container_of(b, struct node_data, node); + + return node_a->key < node_b->key; +} + static int create_and_stash(int idx, int val) { struct map_value *mapval; @@ -113,6 +131,41 @@ long stash_plain(void *ctx) return 0; } +SEC("tc") +long stash_local_with_root(void *ctx) +{ + struct local_with_root *res; + struct map_value *mapval; + struct node_data *n; + int idx = 0; + + mapval = bpf_map_lookup_elem(&some_nodes, &idx); + if (!mapval) + return 1; + + res = bpf_obj_new(typeof(*res)); + if (!res) + return 2; + res->key = 41; + + n = bpf_obj_new(typeof(*n)); + if (!n) { + bpf_obj_drop(res); + return 3; + } + + bpf_spin_lock(&res->l); + bpf_rbtree_add(&res->r, &n->node, less); + bpf_spin_unlock(&res->l); + + res = bpf_kptr_xchg(&mapval->local_root, res); + if (res) { + bpf_obj_drop(res); + return 4; + } + return 0; +} + SEC("tc") long unstash_rb_node(void *ctx) { diff --git a/tools/testing/selftests/bpf/progs/map_in_map_btf.c b/tools/testing/selftests/bpf/progs/map_in_map_btf.c new file mode 100644 index 00000000000000..7a1336d7b16a63 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/map_in_map_btf.c @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2023. Huawei Technologies Co., Ltd */ +#include +#include +#include + +#include "bpf_misc.h" +#include "bpf_experimental.h" + +struct node_data { + __u64 data; + struct bpf_list_node node; +}; + +struct map_value { + struct bpf_list_head head __contains(node_data, node); + struct bpf_spin_lock lock; +}; + +struct inner_array_type { + __uint(type, BPF_MAP_TYPE_ARRAY); + __type(key, int); + __type(value, struct map_value); + __uint(max_entries, 1); +} inner_array SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); + __uint(key_size, 4); + __uint(value_size, 4); + __uint(max_entries, 1); + __array(values, struct inner_array_type); +} outer_array SEC(".maps") = { + .values = { + [0] = &inner_array, + }, +}; + +char _license[] SEC("license") = "GPL"; + +int pid = 0; +bool done = false; + +SEC("fentry/" SYS_PREFIX "sys_nanosleep") +int add_to_list_in_inner_array(void *ctx) +{ + struct map_value *value; + struct node_data *new; + struct bpf_map *map; + int zero = 0; + + if (done || (u32)bpf_get_current_pid_tgid() != pid) + return 0; + + map = bpf_map_lookup_elem(&outer_array, &zero); + if (!map) + return 0; + + value = bpf_map_lookup_elem(map, &zero); + if (!value) + return 0; + + new = bpf_obj_new(typeof(*new)); + if (!new) + return 0; + + bpf_spin_lock(&value->lock); + bpf_list_push_back(&value->head, &new->node); + bpf_spin_unlock(&value->lock); + done = true; + + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/normal_map_btf.c b/tools/testing/selftests/bpf/progs/normal_map_btf.c new file mode 100644 index 00000000000000..66cde82aa86d92 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/normal_map_btf.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2023. Huawei Technologies Co., Ltd */ +#include +#include +#include + +#include "bpf_misc.h" +#include "bpf_experimental.h" + +struct node_data { + __u64 data; + struct bpf_list_node node; +}; + +struct map_value { + struct bpf_list_head head __contains(node_data, node); + struct bpf_spin_lock lock; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __type(key, int); + __type(value, struct map_value); + __uint(max_entries, 1); +} array SEC(".maps"); + +char _license[] SEC("license") = "GPL"; + +int pid = 0; +bool done = false; + +SEC("fentry/" SYS_PREFIX "sys_nanosleep") +int add_to_list_in_array(void *ctx) +{ + struct map_value *value; + struct node_data *new; + int zero = 0; + + if (done || (u32)bpf_get_current_pid_tgid() != pid) + return 0; + + value = bpf_map_lookup_elem(&array, &zero); + if (!value) + return 0; + + new = bpf_obj_new(typeof(*new)); + if (!new) + return 0; + + bpf_spin_lock(&value->lock); + bpf_list_push_back(&value->head, &new->node); + bpf_spin_unlock(&value->lock); + done = true; + + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/priv_map.c b/tools/testing/selftests/bpf/progs/priv_map.c new file mode 100644 index 00000000000000..9085be50f03bf4 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/priv_map.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */ + +#include "vmlinux.h" +#include + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_QUEUE); + __uint(max_entries, 1); + __type(value, __u32); +} priv_map SEC(".maps"); diff --git a/tools/testing/selftests/bpf/progs/priv_prog.c b/tools/testing/selftests/bpf/progs/priv_prog.c new file mode 100644 index 00000000000000..3c7b2b618c8a9b --- /dev/null +++ b/tools/testing/selftests/bpf/progs/priv_prog.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */ + +#include "vmlinux.h" +#include + +char _license[] SEC("license") = "GPL"; + +SEC("kprobe") +int kprobe_prog(void *ctx) +{ + return 1; +} diff --git a/tools/testing/selftests/bpf/progs/syscall.c b/tools/testing/selftests/bpf/progs/syscall.c index e550f728962d40..3d3cafdebe72b0 100644 --- a/tools/testing/selftests/bpf/progs/syscall.c +++ b/tools/testing/selftests/bpf/progs/syscall.c @@ -6,9 +6,15 @@ #include #include <../../../tools/include/linux/filter.h> #include +#include +#include char _license[] SEC("license") = "GPL"; +struct bpf_map { + int id; +} __attribute__((preserve_access_index)); + struct args { __u64 log_buf; __u32 log_size; @@ -27,6 +33,37 @@ struct args { BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_INT, 0, 0), sz), \ BTF_INT_ENC(encoding, bits_offset, bits) +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __type(key, int); + __type(value, union bpf_attr); + __uint(max_entries, 1); +} bpf_attr_array SEC(".maps"); + +struct inner_map_type { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(key_size, 4); + __uint(value_size, 4); + __uint(max_entries, 1); +} inner_map SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); + __type(key, int); + __type(value, int); + __uint(max_entries, 1); + __array(values, struct inner_map_type); +} outer_array_map SEC(".maps") = { + .values = { + [0] = &inner_map, + }, +}; + +static inline __u64 ptr_to_u64(const void *ptr) +{ + return (__u64) (unsigned long) ptr; +} + static int btf_load(void) { struct btf_blob { @@ -58,7 +95,7 @@ static int btf_load(void) } SEC("syscall") -int bpf_prog(struct args *ctx) +int load_prog(struct args *ctx) { static char license[] = "GPL"; static struct bpf_insn insns[] = { @@ -94,8 +131,8 @@ int bpf_prog(struct args *ctx) map_create_attr.max_entries = ctx->max_entries; map_create_attr.btf_fd = ret; - prog_load_attr.license = (long) license; - prog_load_attr.insns = (long) insns; + prog_load_attr.license = ptr_to_u64(license); + prog_load_attr.insns = ptr_to_u64(insns); prog_load_attr.log_buf = ctx->log_buf; prog_load_attr.log_size = ctx->log_size; prog_load_attr.log_level = 1; @@ -107,8 +144,8 @@ int bpf_prog(struct args *ctx) insns[3].imm = ret; map_update_attr.map_fd = ret; - map_update_attr.key = (long) &key; - map_update_attr.value = (long) &value; + map_update_attr.key = ptr_to_u64(&key); + map_update_attr.value = ptr_to_u64(&value); ret = bpf_sys_bpf(BPF_MAP_UPDATE_ELEM, &map_update_attr, sizeof(map_update_attr)); if (ret < 0) return ret; @@ -119,3 +156,52 @@ int bpf_prog(struct args *ctx) ctx->prog_fd = ret; return 1; } + +SEC("syscall") +int update_outer_map(void *ctx) +{ + int zero = 0, ret = 0, outer_fd = -1, inner_fd = -1, err; + const int attr_sz = sizeof(union bpf_attr); + union bpf_attr *attr; + + attr = bpf_map_lookup_elem((struct bpf_map *)&bpf_attr_array, &zero); + if (!attr) + goto out; + + memset(attr, 0, attr_sz); + attr->map_id = ((struct bpf_map *)&outer_array_map)->id; + outer_fd = bpf_sys_bpf(BPF_MAP_GET_FD_BY_ID, attr, attr_sz); + if (outer_fd < 0) + goto out; + + memset(attr, 0, attr_sz); + attr->map_type = BPF_MAP_TYPE_ARRAY; + attr->key_size = 4; + attr->value_size = 4; + attr->max_entries = 1; + inner_fd = bpf_sys_bpf(BPF_MAP_CREATE, attr, attr_sz); + if (inner_fd < 0) + goto out; + + memset(attr, 0, attr_sz); + attr->map_fd = outer_fd; + attr->key = ptr_to_u64(&zero); + attr->value = ptr_to_u64(&inner_fd); + err = bpf_sys_bpf(BPF_MAP_UPDATE_ELEM, attr, attr_sz); + if (err) + goto out; + + memset(attr, 0, attr_sz); + attr->map_fd = outer_fd; + attr->key = ptr_to_u64(&zero); + err = bpf_sys_bpf(BPF_MAP_DELETE_ELEM, attr, attr_sz); + if (err) + goto out; + ret = 1; +out: + if (inner_fd >= 0) + bpf_sys_close(inner_fd); + if (outer_fd >= 0) + bpf_sys_close(outer_fd); + return ret; +} diff --git a/tools/testing/selftests/bpf/progs/test_fsverity.c b/tools/testing/selftests/bpf/progs/test_fsverity.c new file mode 100644 index 00000000000000..3975495b75c8df --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_fsverity.c @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */ + +#include "vmlinux.h" +#include +#include +#include "bpf_kfuncs.h" + +char _license[] SEC("license") = "GPL"; + +#ifndef SHA256_DIGEST_SIZE +#define SHA256_DIGEST_SIZE 32 +#endif + +#define SIZEOF_STRUCT_FSVERITY_DIGEST 4 /* sizeof(struct fsverity_digest) */ + +char expected_digest[SIZEOF_STRUCT_FSVERITY_DIGEST + SHA256_DIGEST_SIZE]; +char digest[SIZEOF_STRUCT_FSVERITY_DIGEST + SHA256_DIGEST_SIZE]; +__u32 monitored_pid; +__u32 got_fsverity; +__u32 digest_matches; + +SEC("lsm.s/file_open") +int BPF_PROG(test_file_open, struct file *f) +{ + struct bpf_dynptr digest_ptr; + __u32 pid; + int ret; + int i; + + pid = bpf_get_current_pid_tgid() >> 32; + if (pid != monitored_pid) + return 0; + + bpf_dynptr_from_mem(digest, sizeof(digest), 0, &digest_ptr); + ret = bpf_get_fsverity_digest(f, &digest_ptr); + if (ret < 0) + return 0; + got_fsverity = 1; + + for (i = 0; i < sizeof(digest); i++) { + if (digest[i] != expected_digest[i]) + return 0; + } + + digest_matches = 1; + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/test_get_xattr.c b/tools/testing/selftests/bpf/progs/test_get_xattr.c new file mode 100644 index 00000000000000..7eb2a4e5a3e576 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_get_xattr.c @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */ + +#include "vmlinux.h" +#include +#include +#include "bpf_kfuncs.h" + +char _license[] SEC("license") = "GPL"; + +__u32 monitored_pid; +__u32 found_xattr; + +static const char expected_value[] = "hello"; +char value[32]; + +SEC("lsm.s/file_open") +int BPF_PROG(test_file_open, struct file *f) +{ + struct bpf_dynptr value_ptr; + __u32 pid; + int ret; + + pid = bpf_get_current_pid_tgid() >> 32; + if (pid != monitored_pid) + return 0; + + bpf_dynptr_from_mem(value, sizeof(value), 0, &value_ptr); + + ret = bpf_get_file_xattr(f, "user.kfuncs", &value_ptr); + if (ret != sizeof(expected_value)) + return 0; + if (bpf_strncmp(value, ret, expected_value)) + return 0; + found_xattr = 1; + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/test_global_func15.c b/tools/testing/selftests/bpf/progs/test_global_func15.c index b512d6a6c75e54..b4e089d6981da0 100644 --- a/tools/testing/selftests/bpf/progs/test_global_func15.c +++ b/tools/testing/selftests/bpf/progs/test_global_func15.c @@ -13,7 +13,7 @@ __noinline int foo(unsigned int *v) } SEC("cgroup_skb/ingress") -__failure __msg("At program exit the register R0 has value") +__failure __msg("At program exit the register R0 has ") int global_func15(struct __sk_buff *skb) { unsigned int v = 1; @@ -22,3 +22,35 @@ int global_func15(struct __sk_buff *skb) return v; } + +SEC("cgroup_skb/ingress") +__log_level(2) __flag(BPF_F_TEST_STATE_FREQ) +__failure +/* check that fallthrough code path marks r0 as precise */ +__msg("mark_precise: frame0: regs=r0 stack= before 2: (b7) r0 = 1") +/* check that branch code path marks r0 as precise */ +__msg("mark_precise: frame0: regs=r0 stack= before 0: (85) call bpf_get_prandom_u32#7") +__msg("At program exit the register R0 has ") +__naked int global_func15_tricky_pruning(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "if r0 s> 1000 goto 1f;" + "r0 = 1;" + "1:" + "goto +0;" /* checkpoint */ + /* cgroup_skb/ingress program is expected to return [0, 1] + * values, so branch above makes sure that in a fallthrough + * case we have a valid 1 stored in R0 register, but in + * a branch case we assign some random value to R0. So if + * there is something wrong with precision tracking for R0 at + * program exit, we might erronenously prune branch case, + * because R0 in fallthrough case is imprecise (and thus any + * value is valid from POV of verifier is_state_equal() logic) + */ + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_common + ); +} diff --git a/tools/testing/selftests/bpf/progs/test_global_func16.c b/tools/testing/selftests/bpf/progs/test_global_func16.c index e7206304632e15..e3e64bc472cdaf 100644 --- a/tools/testing/selftests/bpf/progs/test_global_func16.c +++ b/tools/testing/selftests/bpf/progs/test_global_func16.c @@ -13,7 +13,7 @@ __noinline int foo(int (*arr)[10]) } SEC("cgroup_skb/ingress") -__failure __msg("invalid indirect read from stack") +__success int global_func16(struct __sk_buff *skb) { int array[10]; diff --git a/tools/testing/selftests/bpf/progs/test_sig_in_xattr.c b/tools/testing/selftests/bpf/progs/test_sig_in_xattr.c new file mode 100644 index 00000000000000..2f0eb1334d65a3 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_sig_in_xattr.c @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */ + +#include "vmlinux.h" +#include +#include +#include +#include "bpf_kfuncs.h" + +char _license[] SEC("license") = "GPL"; + +#ifndef SHA256_DIGEST_SIZE +#define SHA256_DIGEST_SIZE 32 +#endif + +#define MAX_SIG_SIZE 1024 + +/* By default, "fsverity sign" signs a file with fsverity_formatted_digest + * of the file. fsverity_formatted_digest on the kernel side is only used + * with CONFIG_FS_VERITY_BUILTIN_SIGNATURES. However, BPF LSM doesn't not + * require CONFIG_FS_VERITY_BUILTIN_SIGNATURES, so vmlinux.h may not have + * fsverity_formatted_digest. In this test, we intentionally avoid using + * fsverity_formatted_digest. + * + * Luckily, fsverity_formatted_digest is simply 8-byte magic followed by + * fsverity_digest. We use a char array of size fsverity_formatted_digest + * plus SHA256_DIGEST_SIZE. The magic part of it is filled by user space, + * and the rest of it is filled by bpf_get_fsverity_digest. + * + * Note that, generating signatures based on fsverity_formatted_digest is + * the design choice of this selftest (and "fsverity sign"). With BPF + * LSM, we have the flexibility to generate signature based on other data + * sets, for example, fsverity_digest or only the digest[] part of it. + */ +#define MAGIC_SIZE 8 +#define SIZEOF_STRUCT_FSVERITY_DIGEST 4 /* sizeof(struct fsverity_digest) */ +char digest[MAGIC_SIZE + SIZEOF_STRUCT_FSVERITY_DIGEST + SHA256_DIGEST_SIZE]; + +__u32 monitored_pid; +char sig[MAX_SIG_SIZE]; +__u32 sig_size; +__u32 user_keyring_serial; + +SEC("lsm.s/file_open") +int BPF_PROG(test_file_open, struct file *f) +{ + struct bpf_dynptr digest_ptr, sig_ptr; + struct bpf_key *trusted_keyring; + __u32 pid; + int ret; + + pid = bpf_get_current_pid_tgid() >> 32; + if (pid != monitored_pid) + return 0; + + /* digest_ptr points to fsverity_digest */ + bpf_dynptr_from_mem(digest + MAGIC_SIZE, sizeof(digest) - MAGIC_SIZE, 0, &digest_ptr); + + ret = bpf_get_fsverity_digest(f, &digest_ptr); + /* No verity, allow access */ + if (ret < 0) + return 0; + + /* Move digest_ptr to fsverity_formatted_digest */ + bpf_dynptr_from_mem(digest, sizeof(digest), 0, &digest_ptr); + + /* Read signature from xattr */ + bpf_dynptr_from_mem(sig, sizeof(sig), 0, &sig_ptr); + ret = bpf_get_file_xattr(f, "user.sig", &sig_ptr); + /* No signature, reject access */ + if (ret < 0) + return -EPERM; + + trusted_keyring = bpf_lookup_user_key(user_keyring_serial, 0); + if (!trusted_keyring) + return -ENOENT; + + /* Verify signature */ + ret = bpf_verify_pkcs7_signature(&digest_ptr, &sig_ptr, trusted_keyring); + + bpf_key_put(trusted_keyring); + return ret; +} diff --git a/tools/testing/selftests/bpf/progs/test_tunnel_kern.c b/tools/testing/selftests/bpf/progs/test_tunnel_kern.c index f66af753bbbb89..3e436e6f731251 100644 --- a/tools/testing/selftests/bpf/progs/test_tunnel_kern.c +++ b/tools/testing/selftests/bpf/progs/test_tunnel_kern.c @@ -6,66 +6,34 @@ * modify it under the terms of version 2 of the GNU General Public * License as published by the Free Software Foundation. */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include "vmlinux.h" +#include #include #include +#include "bpf_kfuncs.h" +#include "bpf_tracing_net.h" #define log_err(__ret) bpf_printk("ERROR line:%d ret:%d\n", __LINE__, __ret) -#define VXLAN_UDP_PORT 4789 +#define VXLAN_UDP_PORT 4789 +#define ETH_P_IP 0x0800 +#define PACKET_HOST 0 +#define TUNNEL_CSUM bpf_htons(0x01) +#define TUNNEL_KEY bpf_htons(0x04) /* Only IPv4 address assigned to veth1. * 172.16.1.200 */ #define ASSIGNED_ADDR_VETH1 0xac1001c8 -struct geneve_opt { - __be16 opt_class; - __u8 type; - __u8 length:5; - __u8 r3:1; - __u8 r2:1; - __u8 r1:1; - __u8 opt_data[8]; /* hard-coded to 8 byte */ -}; - -struct vxlanhdr { - __be32 vx_flags; - __be32 vx_vni; -} __attribute__((packed)); - -struct vxlan_metadata { - __u32 gbp; -}; - -struct bpf_fou_encap { - __be16 sport; - __be16 dport; -}; - -enum bpf_fou_encap_type { - FOU_BPF_ENCAP_FOU, - FOU_BPF_ENCAP_GUE, -}; - int bpf_skb_set_fou_encap(struct __sk_buff *skb_ctx, struct bpf_fou_encap *encap, int type) __ksym; int bpf_skb_get_fou_encap(struct __sk_buff *skb_ctx, struct bpf_fou_encap *encap) __ksym; +struct xfrm_state * +bpf_xdp_get_xfrm_state(struct xdp_md *ctx, struct bpf_xfrm_state_opts *opts, + u32 opts__sz) __ksym; +void bpf_xdp_xfrm_state_release(struct xfrm_state *x) __ksym; struct { __uint(type, BPF_MAP_TYPE_ARRAY); @@ -205,9 +173,9 @@ int erspan_set_tunnel(struct __sk_buff *skb) __u8 hwid = 7; md.version = 2; - md.u.md2.dir = direction; - md.u.md2.hwid = hwid & 0xf; - md.u.md2.hwid_upper = (hwid >> 4) & 0x3; + BPF_CORE_WRITE_BITFIELD(&md.u.md2, dir, direction); + BPF_CORE_WRITE_BITFIELD(&md.u.md2, hwid, (hwid & 0xf)); + BPF_CORE_WRITE_BITFIELD(&md.u.md2, hwid_upper, (hwid >> 4) & 0x3); #endif ret = bpf_skb_set_tunnel_opt(skb, &md, sizeof(md)); @@ -246,8 +214,9 @@ int erspan_get_tunnel(struct __sk_buff *skb) bpf_printk("\tindex %x\n", index); #else bpf_printk("\tdirection %d hwid %x timestamp %u\n", - md.u.md2.dir, - (md.u.md2.hwid_upper << 4) + md.u.md2.hwid, + BPF_CORE_READ_BITFIELD(&md.u.md2, dir), + (BPF_CORE_READ_BITFIELD(&md.u.md2, hwid_upper) << 4) + + BPF_CORE_READ_BITFIELD(&md.u.md2, hwid), bpf_ntohl(md.u.md2.timestamp)); #endif @@ -284,9 +253,9 @@ int ip4ip6erspan_set_tunnel(struct __sk_buff *skb) __u8 hwid = 17; md.version = 2; - md.u.md2.dir = direction; - md.u.md2.hwid = hwid & 0xf; - md.u.md2.hwid_upper = (hwid >> 4) & 0x3; + BPF_CORE_WRITE_BITFIELD(&md.u.md2, dir, direction); + BPF_CORE_WRITE_BITFIELD(&md.u.md2, hwid, (hwid & 0xf)); + BPF_CORE_WRITE_BITFIELD(&md.u.md2, hwid_upper, (hwid >> 4) & 0x3); #endif ret = bpf_skb_set_tunnel_opt(skb, &md, sizeof(md)); @@ -326,8 +295,9 @@ int ip4ip6erspan_get_tunnel(struct __sk_buff *skb) bpf_printk("\tindex %x\n", index); #else bpf_printk("\tdirection %d hwid %x timestamp %u\n", - md.u.md2.dir, - (md.u.md2.hwid_upper << 4) + md.u.md2.hwid, + BPF_CORE_READ_BITFIELD(&md.u.md2, dir), + (BPF_CORE_READ_BITFIELD(&md.u.md2, hwid_upper) << 4) + + BPF_CORE_READ_BITFIELD(&md.u.md2, hwid), bpf_ntohl(md.u.md2.timestamp)); #endif @@ -963,6 +933,10 @@ int ip6ip6_get_tunnel(struct __sk_buff *skb) return TC_ACT_OK; } +volatile int xfrm_reqid = 0; +volatile int xfrm_spi = 0; +volatile int xfrm_remote_ip = 0; + SEC("tc") int xfrm_get_state(struct __sk_buff *skb) { @@ -973,10 +947,58 @@ int xfrm_get_state(struct __sk_buff *skb) if (ret < 0) return TC_ACT_OK; - bpf_printk("reqid %d spi 0x%x remote ip 0x%x\n", - x.reqid, bpf_ntohl(x.spi), - bpf_ntohl(x.remote_ipv4)); + xfrm_reqid = x.reqid; + xfrm_spi = bpf_ntohl(x.spi); + xfrm_remote_ip = bpf_ntohl(x.remote_ipv4); + return TC_ACT_OK; } +volatile int xfrm_replay_window = 0; + +SEC("xdp") +int xfrm_get_state_xdp(struct xdp_md *xdp) +{ + struct bpf_xfrm_state_opts opts = {}; + struct xfrm_state *x = NULL; + struct ip_esp_hdr *esph; + struct bpf_dynptr ptr; + u8 esph_buf[8] = {}; + u8 iph_buf[20] = {}; + struct iphdr *iph; + u32 off; + + if (bpf_dynptr_from_xdp(xdp, 0, &ptr)) + goto out; + + off = sizeof(struct ethhdr); + iph = bpf_dynptr_slice(&ptr, off, iph_buf, sizeof(iph_buf)); + if (!iph || iph->protocol != IPPROTO_ESP) + goto out; + + off += sizeof(struct iphdr); + esph = bpf_dynptr_slice(&ptr, off, esph_buf, sizeof(esph_buf)); + if (!esph) + goto out; + + opts.netns_id = BPF_F_CURRENT_NETNS; + opts.daddr.a4 = iph->daddr; + opts.spi = esph->spi; + opts.proto = IPPROTO_ESP; + opts.family = AF_INET; + + x = bpf_xdp_get_xfrm_state(xdp, &opts, sizeof(opts)); + if (!x) + goto out; + + if (!x->replay_esn) + goto out; + + xfrm_replay_window = x->replay_esn->replay_window; +out: + if (x) + bpf_xdp_xfrm_state_release(x); + return XDP_PASS; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_verify_pkcs7_sig.c b/tools/testing/selftests/bpf/progs/test_verify_pkcs7_sig.c index 7748cc23de8a90..f42e9f3831a126 100644 --- a/tools/testing/selftests/bpf/progs/test_verify_pkcs7_sig.c +++ b/tools/testing/selftests/bpf/progs/test_verify_pkcs7_sig.c @@ -10,17 +10,11 @@ #include #include #include +#include "bpf_kfuncs.h" #define MAX_DATA_SIZE (1024 * 1024) #define MAX_SIG_SIZE 1024 -extern struct bpf_key *bpf_lookup_user_key(__u32 serial, __u64 flags) __ksym; -extern struct bpf_key *bpf_lookup_system_key(__u64 id) __ksym; -extern void bpf_key_put(struct bpf_key *key) __ksym; -extern int bpf_verify_pkcs7_signature(struct bpf_dynptr *data_ptr, - struct bpf_dynptr *sig_ptr, - struct bpf_key *trusted_keyring) __ksym; - __u32 monitored_pid; __u32 user_keyring_serial; __u64 system_keyring_id; diff --git a/tools/testing/selftests/bpf/progs/timer_failure.c b/tools/testing/selftests/bpf/progs/timer_failure.c index 226d33b5a05c80..0996c2486f0562 100644 --- a/tools/testing/selftests/bpf/progs/timer_failure.c +++ b/tools/testing/selftests/bpf/progs/timer_failure.c @@ -21,17 +21,38 @@ struct { __type(value, struct elem); } timer_map SEC(".maps"); -static int timer_cb_ret1(void *map, int *key, struct bpf_timer *timer) +__naked __noinline __used +static unsigned long timer_cb_ret_bad() { - if (bpf_get_smp_processor_id() % 2) - return 1; - else - return 0; + asm volatile ( + "call %[bpf_get_prandom_u32];" + "if r0 s> 1000 goto 1f;" + "r0 = 0;" + "1:" + "goto +0;" /* checkpoint */ + /* async callback is expected to return 0, so branch above + * skipping r0 = 0; should lead to a failure, but if exit + * instruction doesn't enforce r0's precision, this callback + * will be successfully verified + */ + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_common + ); } SEC("fentry/bpf_fentry_test1") -__failure __msg("should have been in (0x0; 0x0)") -int BPF_PROG2(test_ret_1, int, a) +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__failure +/* check that fallthrough code path marks r0 as precise */ +__msg("mark_precise: frame0: regs=r0 stack= before") +__msg(": (85) call bpf_get_prandom_u32#7") /* anchor message */ +/* check that branch code path marks r0 as precise */ +__msg("mark_precise: frame0: regs=r0 stack= before ") __msg(": (85) call bpf_get_prandom_u32#7") +__msg("should have been in [0, 0]") +long BPF_PROG2(test_bad_ret, int, a) { int key = 0; struct bpf_timer *timer; @@ -39,7 +60,7 @@ int BPF_PROG2(test_ret_1, int, a) timer = bpf_map_lookup_elem(&timer_map, &key); if (timer) { bpf_timer_init(timer, &timer_map, CLOCK_BOOTTIME); - bpf_timer_set_callback(timer, timer_cb_ret1); + bpf_timer_set_callback(timer, timer_cb_ret_bad); bpf_timer_start(timer, 1000, 0); } diff --git a/tools/testing/selftests/bpf/progs/user_ringbuf_fail.c b/tools/testing/selftests/bpf/progs/user_ringbuf_fail.c index 03ee946c6bf76a..11ab25c42c360a 100644 --- a/tools/testing/selftests/bpf/progs/user_ringbuf_fail.c +++ b/tools/testing/selftests/bpf/progs/user_ringbuf_fail.c @@ -184,7 +184,7 @@ invalid_drain_callback_return(struct bpf_dynptr *dynptr, void *context) * not be able to write to that pointer. */ SEC("?raw_tp") -__failure __msg("At callback return the register R0 has value") +__failure __msg("At callback return the register R0 has ") int user_ringbuf_callback_invalid_return(void *ctx) { bpf_user_ringbuf_drain(&user_ringbuf, invalid_drain_callback_return, NULL, 0); diff --git a/tools/testing/selftests/bpf/progs/verifier_basic_stack.c b/tools/testing/selftests/bpf/progs/verifier_basic_stack.c index 359df865a8f3e9..8d77cc5323d337 100644 --- a/tools/testing/selftests/bpf/progs/verifier_basic_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_basic_stack.c @@ -27,8 +27,8 @@ __naked void stack_out_of_bounds(void) SEC("socket") __description("uninitialized stack1") -__failure __msg("invalid indirect read from stack") -__failure_unpriv +__success __log_level(4) __msg("stack depth 8") +__failure_unpriv __msg_unpriv("invalid indirect read from stack") __naked void uninitialized_stack1(void) { asm volatile (" \ @@ -45,8 +45,8 @@ __naked void uninitialized_stack1(void) SEC("socket") __description("uninitialized stack2") -__failure __msg("invalid read from stack") -__failure_unpriv +__success __log_level(4) __msg("stack depth 8") +__failure_unpriv __msg_unpriv("invalid read from stack") __naked void uninitialized_stack2(void) { asm volatile (" \ diff --git a/tools/testing/selftests/bpf/progs/verifier_bitfield_write.c b/tools/testing/selftests/bpf/progs/verifier_bitfield_write.c new file mode 100644 index 00000000000000..623f130a319891 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_bitfield_write.c @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +#include +#include + +#include "bpf_misc.h" + +struct core_reloc_bitfields { + /* unsigned bitfields */ + uint8_t ub1: 1; + uint8_t ub2: 2; + uint32_t ub7: 7; + /* signed bitfields */ + int8_t sb4: 4; + int32_t sb20: 20; + /* non-bitfields */ + uint32_t u32; + int32_t s32; +} __attribute__((preserve_access_index)); + +SEC("tc") +__description("single CO-RE bitfield roundtrip") +__btf_path("btf__core_reloc_bitfields.bpf.o") +__success +__retval(3) +int single_field_roundtrip(struct __sk_buff *ctx) +{ + struct core_reloc_bitfields bitfields; + + __builtin_memset(&bitfields, 0, sizeof(bitfields)); + BPF_CORE_WRITE_BITFIELD(&bitfields, ub2, 3); + return BPF_CORE_READ_BITFIELD(&bitfields, ub2); +} + +SEC("tc") +__description("multiple CO-RE bitfield roundtrip") +__btf_path("btf__core_reloc_bitfields.bpf.o") +__success +__retval(0x3FD) +int multiple_field_roundtrip(struct __sk_buff *ctx) +{ + struct core_reloc_bitfields bitfields; + uint8_t ub2; + int8_t sb4; + + __builtin_memset(&bitfields, 0, sizeof(bitfields)); + BPF_CORE_WRITE_BITFIELD(&bitfields, ub2, 1); + BPF_CORE_WRITE_BITFIELD(&bitfields, sb4, -1); + + ub2 = BPF_CORE_READ_BITFIELD(&bitfields, ub2); + sb4 = BPF_CORE_READ_BITFIELD(&bitfields, sb4); + + return (((uint8_t)sb4) << 2) | ub2; +} + +SEC("tc") +__description("adjacent CO-RE bitfield roundtrip") +__btf_path("btf__core_reloc_bitfields.bpf.o") +__success +__retval(7) +int adjacent_field_roundtrip(struct __sk_buff *ctx) +{ + struct core_reloc_bitfields bitfields; + uint8_t ub1, ub2; + + __builtin_memset(&bitfields, 0, sizeof(bitfields)); + BPF_CORE_WRITE_BITFIELD(&bitfields, ub1, 1); + BPF_CORE_WRITE_BITFIELD(&bitfields, ub2, 3); + + ub1 = BPF_CORE_READ_BITFIELD(&bitfields, ub1); + ub2 = BPF_CORE_READ_BITFIELD(&bitfields, ub2); + + return (ub2 << 1) | ub1; +} + +SEC("tc") +__description("multibyte CO-RE bitfield roundtrip") +__btf_path("btf__core_reloc_bitfields.bpf.o") +__success +__retval(0x21) +int multibyte_field_roundtrip(struct __sk_buff *ctx) +{ + struct core_reloc_bitfields bitfields; + uint32_t ub7; + uint8_t ub1; + + __builtin_memset(&bitfields, 0, sizeof(bitfields)); + BPF_CORE_WRITE_BITFIELD(&bitfields, ub1, 1); + BPF_CORE_WRITE_BITFIELD(&bitfields, ub7, 16); + + ub1 = BPF_CORE_READ_BITFIELD(&bitfields, ub1); + ub7 = BPF_CORE_READ_BITFIELD(&bitfields, ub7); + + return (ub7 << 1) | ub1; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_cgroup_inv_retcode.c b/tools/testing/selftests/bpf/progs/verifier_cgroup_inv_retcode.c index d6c4a7f3f7900e..6e0f349f8f15fa 100644 --- a/tools/testing/selftests/bpf/progs/verifier_cgroup_inv_retcode.c +++ b/tools/testing/selftests/bpf/progs/verifier_cgroup_inv_retcode.c @@ -7,7 +7,7 @@ SEC("cgroup/sock") __description("bpf_exit with invalid return code. test1") -__failure __msg("R0 has value (0x0; 0xffffffff)") +__failure __msg("smin=0 smax=4294967295 should have been in [0, 1]") __naked void with_invalid_return_code_test1(void) { asm volatile (" \ @@ -30,7 +30,7 @@ __naked void with_invalid_return_code_test2(void) SEC("cgroup/sock") __description("bpf_exit with invalid return code. test3") -__failure __msg("R0 has value (0x0; 0x3)") +__failure __msg("smin=0 smax=3 should have been in [0, 1]") __naked void with_invalid_return_code_test3(void) { asm volatile (" \ @@ -53,7 +53,7 @@ __naked void with_invalid_return_code_test4(void) SEC("cgroup/sock") __description("bpf_exit with invalid return code. test5") -__failure __msg("R0 has value (0x2; 0x0)") +__failure __msg("smin=2 smax=2 should have been in [0, 1]") __naked void with_invalid_return_code_test5(void) { asm volatile (" \ @@ -75,7 +75,7 @@ __naked void with_invalid_return_code_test6(void) SEC("cgroup/sock") __description("bpf_exit with invalid return code. test7") -__failure __msg("R0 has unknown scalar value") +__failure __msg("R0 has unknown scalar value should have been in [0, 1]") __naked void with_invalid_return_code_test7(void) { asm volatile (" \ diff --git a/tools/testing/selftests/bpf/progs/verifier_direct_packet_access.c b/tools/testing/selftests/bpf/progs/verifier_direct_packet_access.c index 99a23dea8233fe..be95570ab382e3 100644 --- a/tools/testing/selftests/bpf/progs/verifier_direct_packet_access.c +++ b/tools/testing/selftests/bpf/progs/verifier_direct_packet_access.c @@ -411,7 +411,7 @@ l0_%=: r0 = 0; \ SEC("tc") __description("direct packet access: test17 (pruning, alignment)") -__failure __msg("misaligned packet access off 2+(0x0; 0x0)+15+-4 size 4") +__failure __msg("misaligned packet access off 2+0+15+-4 size 4") __flag(BPF_F_STRICT_ALIGNMENT) __naked void packet_access_test17_pruning_alignment(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c b/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c index a0a5efd1caa106..bd696a43124481 100644 --- a/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c +++ b/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c @@ -10,6 +10,7 @@ int arr[1]; int unkn_idx; +const volatile bool call_dead_subprog = false; __noinline long global_bad(void) { @@ -31,23 +32,31 @@ __noinline long global_calls_good_only(void) return global_good(); } +__noinline long global_dead(void) +{ + return arr[0] * 2; +} + SEC("?raw_tp") __success __log_level(2) /* main prog is validated completely first */ __msg("('global_calls_good_only') is global and assumed valid.") -__msg("1: (95) exit") /* eventually global_good() is transitively validated as well */ __msg("Validating global_good() func") __msg("('global_good') is safe for any args that match its prototype") int chained_global_func_calls_success(void) { - return global_calls_good_only(); + int sum = 0; + + if (call_dead_subprog) + sum += global_dead(); + return global_calls_good_only() + sum; } SEC("?raw_tp") __failure __log_level(2) /* main prog validated successfully first */ -__msg("1: (95) exit") +__msg("('global_calls_bad') is global and assumed valid.") /* eventually we validate global_bad() and fail */ __msg("Validating global_bad() func") __msg("math between map_value pointer and register") /* BOOM */ diff --git a/tools/testing/selftests/bpf/progs/verifier_int_ptr.c b/tools/testing/selftests/bpf/progs/verifier_int_ptr.c index b054f9c4814337..9fc3fae5cd833b 100644 --- a/tools/testing/selftests/bpf/progs/verifier_int_ptr.c +++ b/tools/testing/selftests/bpf/progs/verifier_int_ptr.c @@ -5,9 +5,10 @@ #include #include "bpf_misc.h" -SEC("cgroup/sysctl") +SEC("socket") __description("ARG_PTR_TO_LONG uninitialized") -__failure __msg("invalid indirect read from stack R4 off -16+0 size 8") +__success +__failure_unpriv __msg_unpriv("invalid indirect read from stack R4 off -16+0 size 8") __naked void arg_ptr_to_long_uninitialized(void) { asm volatile (" \ @@ -67,7 +68,7 @@ __naked void ptr_to_long_half_uninitialized(void) SEC("cgroup/sysctl") __description("ARG_PTR_TO_LONG misaligned") -__failure __msg("misaligned stack access off (0x0; 0x0)+-20+0 size 8") +__failure __msg("misaligned stack access off 0+-20+0 size 8") __naked void arg_ptr_to_long_misaligned(void) { asm volatile (" \ diff --git a/tools/testing/selftests/bpf/progs/verifier_netfilter_retcode.c b/tools/testing/selftests/bpf/progs/verifier_netfilter_retcode.c index 353ae6da00e17e..e1ffa5d32ff064 100644 --- a/tools/testing/selftests/bpf/progs/verifier_netfilter_retcode.c +++ b/tools/testing/selftests/bpf/progs/verifier_netfilter_retcode.c @@ -39,7 +39,7 @@ __naked void with_valid_return_code_test3(void) SEC("netfilter") __description("bpf_exit with invalid return code. test4") -__failure __msg("R0 has value (0x2; 0x0)") +__failure __msg("R0 has smin=2 smax=2 should have been in [0, 1]") __naked void with_invalid_return_code_test4(void) { asm volatile (" \ diff --git a/tools/testing/selftests/bpf/progs/verifier_raw_stack.c b/tools/testing/selftests/bpf/progs/verifier_raw_stack.c index efbfc3a4ad6a99..f67390224a9cf9 100644 --- a/tools/testing/selftests/bpf/progs/verifier_raw_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_raw_stack.c @@ -5,9 +5,10 @@ #include #include "bpf_misc.h" -SEC("tc") +SEC("socket") __description("raw_stack: no skb_load_bytes") -__failure __msg("invalid read from stack R6 off=-8 size=8") +__success +__failure_unpriv __msg_unpriv("invalid read from stack R6 off=-8 size=8") __naked void stack_no_skb_load_bytes(void) { asm volatile (" \ diff --git a/tools/testing/selftests/bpf/progs/verifier_spill_fill.c b/tools/testing/selftests/bpf/progs/verifier_spill_fill.c index 6115520154e330..39fe3372e0e0e7 100644 --- a/tools/testing/selftests/bpf/progs/verifier_spill_fill.c +++ b/tools/testing/selftests/bpf/progs/verifier_spill_fill.c @@ -4,6 +4,7 @@ #include #include #include "bpf_misc.h" +#include <../../../tools/include/linux/filter.h> struct { __uint(type, BPF_MAP_TYPE_RINGBUF); @@ -450,4 +451,290 @@ l0_%=: r1 >>= 16; \ : __clobber_all); } +SEC("raw_tp") +__log_level(2) +__success +__msg("fp-8=0m??mmmm") +__msg("fp-16=00mm??mm") +__msg("fp-24=00mm???m") +__naked void spill_subregs_preserve_stack_zero(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + + /* 32-bit subreg spill with ZERO, MISC, and INVALID */ + ".8byte %[fp1_u8_st_zero];" /* ZERO, LLVM-18+: *(u8 *)(r10 -1) = 0; */ + "*(u8 *)(r10 -2) = r0;" /* MISC */ + /* fp-3 and fp-4 stay INVALID */ + "*(u32 *)(r10 -8) = r0;" + + /* 16-bit subreg spill with ZERO, MISC, and INVALID */ + ".8byte %[fp10_u16_st_zero];" /* ZERO, LLVM-18+: *(u16 *)(r10 -10) = 0; */ + "*(u16 *)(r10 -12) = r0;" /* MISC */ + /* fp-13 and fp-14 stay INVALID */ + "*(u16 *)(r10 -16) = r0;" + + /* 8-bit subreg spill with ZERO, MISC, and INVALID */ + ".8byte %[fp18_u16_st_zero];" /* ZERO, LLVM-18+: *(u16 *)(r18 -10) = 0; */ + "*(u16 *)(r10 -20) = r0;" /* MISC */ + /* fp-21, fp-22, and fp-23 stay INVALID */ + "*(u8 *)(r10 -24) = r0;" + + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm_insn(fp1_u8_st_zero, BPF_ST_MEM(BPF_B, BPF_REG_FP, -1, 0)), + __imm_insn(fp10_u16_st_zero, BPF_ST_MEM(BPF_H, BPF_REG_FP, -10, 0)), + __imm_insn(fp18_u16_st_zero, BPF_ST_MEM(BPF_H, BPF_REG_FP, -18, 0)) + : __clobber_all); +} + +char single_byte_buf[1] SEC(".data.single_byte_buf"); + +SEC("raw_tp") +__log_level(2) +__success +/* make sure fp-8 is all STACK_ZERO */ +__msg("2: (7a) *(u64 *)(r10 -8) = 0 ; R10=fp0 fp-8_w=00000000") +/* but fp-16 is spilled IMPRECISE zero const reg */ +__msg("4: (7b) *(u64 *)(r10 -16) = r0 ; R0_w=0 R10=fp0 fp-16_w=0") +/* validate that assigning R2 from STACK_ZERO doesn't mark register + * precise immediately; if necessary, it will be marked precise later + */ +__msg("6: (71) r2 = *(u8 *)(r10 -1) ; R2_w=0 R10=fp0 fp-8_w=00000000") +/* similarly, when R2 is assigned from spilled register, it is initially + * imprecise, but will be marked precise later once it is used in precise context + */ +__msg("10: (71) r2 = *(u8 *)(r10 -9) ; R2_w=0 R10=fp0 fp-16_w=0") +__msg("11: (0f) r1 += r2") +__msg("mark_precise: frame0: last_idx 11 first_idx 0 subseq_idx -1") +__msg("mark_precise: frame0: regs=r2 stack= before 10: (71) r2 = *(u8 *)(r10 -9)") +__msg("mark_precise: frame0: regs= stack=-16 before 9: (bf) r1 = r6") +__msg("mark_precise: frame0: regs= stack=-16 before 8: (73) *(u8 *)(r1 +0) = r2") +__msg("mark_precise: frame0: regs= stack=-16 before 7: (0f) r1 += r2") +__msg("mark_precise: frame0: regs= stack=-16 before 6: (71) r2 = *(u8 *)(r10 -1)") +__msg("mark_precise: frame0: regs= stack=-16 before 5: (bf) r1 = r6") +__msg("mark_precise: frame0: regs= stack=-16 before 4: (7b) *(u64 *)(r10 -16) = r0") +__msg("mark_precise: frame0: regs=r0 stack= before 3: (b7) r0 = 0") +__naked void partial_stack_load_preserves_zeros(void) +{ + asm volatile ( + /* fp-8 is all STACK_ZERO */ + ".8byte %[fp8_st_zero];" /* LLVM-18+: *(u64 *)(r10 -8) = 0; */ + + /* fp-16 is const zero register */ + "r0 = 0;" + "*(u64 *)(r10 -16) = r0;" + + /* load single U8 from non-aligned STACK_ZERO slot */ + "r1 = %[single_byte_buf];" + "r2 = *(u8 *)(r10 -1);" + "r1 += r2;" + "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ + + /* load single U8 from non-aligned ZERO REG slot */ + "r1 = %[single_byte_buf];" + "r2 = *(u8 *)(r10 -9);" + "r1 += r2;" + "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ + + /* load single U16 from non-aligned STACK_ZERO slot */ + "r1 = %[single_byte_buf];" + "r2 = *(u16 *)(r10 -2);" + "r1 += r2;" + "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ + + /* load single U16 from non-aligned ZERO REG slot */ + "r1 = %[single_byte_buf];" + "r2 = *(u16 *)(r10 -10);" + "r1 += r2;" + "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ + + /* load single U32 from non-aligned STACK_ZERO slot */ + "r1 = %[single_byte_buf];" + "r2 = *(u32 *)(r10 -4);" + "r1 += r2;" + "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ + + /* load single U32 from non-aligned ZERO REG slot */ + "r1 = %[single_byte_buf];" + "r2 = *(u32 *)(r10 -12);" + "r1 += r2;" + "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ + + /* for completeness, load U64 from STACK_ZERO slot */ + "r1 = %[single_byte_buf];" + "r2 = *(u64 *)(r10 -8);" + "r1 += r2;" + "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ + + /* for completeness, load U64 from ZERO REG slot */ + "r1 = %[single_byte_buf];" + "r2 = *(u64 *)(r10 -16);" + "r1 += r2;" + "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ + + "r0 = 0;" + "exit;" + : + : __imm_ptr(single_byte_buf), + __imm_insn(fp8_st_zero, BPF_ST_MEM(BPF_DW, BPF_REG_FP, -8, 0)) + : __clobber_common); +} + +char two_byte_buf[2] SEC(".data.two_byte_buf"); + +SEC("raw_tp") +__log_level(2) __flag(BPF_F_TEST_STATE_FREQ) +__success +/* make sure fp-8 is IMPRECISE fake register spill */ +__msg("3: (7a) *(u64 *)(r10 -8) = 1 ; R10=fp0 fp-8_w=1") +/* and fp-16 is spilled IMPRECISE const reg */ +__msg("5: (7b) *(u64 *)(r10 -16) = r0 ; R0_w=1 R10=fp0 fp-16_w=1") +/* validate load from fp-8, which was initialized using BPF_ST_MEM */ +__msg("8: (79) r2 = *(u64 *)(r10 -8) ; R2_w=1 R10=fp0 fp-8=1") +__msg("9: (0f) r1 += r2") +__msg("mark_precise: frame0: last_idx 9 first_idx 7 subseq_idx -1") +__msg("mark_precise: frame0: regs=r2 stack= before 8: (79) r2 = *(u64 *)(r10 -8)") +__msg("mark_precise: frame0: regs= stack=-8 before 7: (bf) r1 = r6") +/* note, fp-8 is precise, fp-16 is not yet precise, we'll get there */ +__msg("mark_precise: frame0: parent state regs= stack=-8: R0_w=1 R1=ctx() R6_r=map_value(map=.data.two_byte_,ks=4,vs=2) R10=fp0 fp-8_rw=P1 fp-16_w=1") +__msg("mark_precise: frame0: last_idx 6 first_idx 3 subseq_idx 7") +__msg("mark_precise: frame0: regs= stack=-8 before 6: (05) goto pc+0") +__msg("mark_precise: frame0: regs= stack=-8 before 5: (7b) *(u64 *)(r10 -16) = r0") +__msg("mark_precise: frame0: regs= stack=-8 before 4: (b7) r0 = 1") +__msg("mark_precise: frame0: regs= stack=-8 before 3: (7a) *(u64 *)(r10 -8) = 1") +__msg("10: R1_w=map_value(map=.data.two_byte_,ks=4,vs=2,off=1) R2_w=1") +/* validate load from fp-16, which was initialized using BPF_STX_MEM */ +__msg("12: (79) r2 = *(u64 *)(r10 -16) ; R2_w=1 R10=fp0 fp-16=1") +__msg("13: (0f) r1 += r2") +__msg("mark_precise: frame0: last_idx 13 first_idx 7 subseq_idx -1") +__msg("mark_precise: frame0: regs=r2 stack= before 12: (79) r2 = *(u64 *)(r10 -16)") +__msg("mark_precise: frame0: regs= stack=-16 before 11: (bf) r1 = r6") +__msg("mark_precise: frame0: regs= stack=-16 before 10: (73) *(u8 *)(r1 +0) = r2") +__msg("mark_precise: frame0: regs= stack=-16 before 9: (0f) r1 += r2") +__msg("mark_precise: frame0: regs= stack=-16 before 8: (79) r2 = *(u64 *)(r10 -8)") +__msg("mark_precise: frame0: regs= stack=-16 before 7: (bf) r1 = r6") +/* now both fp-8 and fp-16 are precise, very good */ +__msg("mark_precise: frame0: parent state regs= stack=-16: R0_w=1 R1=ctx() R6_r=map_value(map=.data.two_byte_,ks=4,vs=2) R10=fp0 fp-8_rw=P1 fp-16_rw=P1") +__msg("mark_precise: frame0: last_idx 6 first_idx 3 subseq_idx 7") +__msg("mark_precise: frame0: regs= stack=-16 before 6: (05) goto pc+0") +__msg("mark_precise: frame0: regs= stack=-16 before 5: (7b) *(u64 *)(r10 -16) = r0") +__msg("mark_precise: frame0: regs=r0 stack= before 4: (b7) r0 = 1") +__msg("14: R1_w=map_value(map=.data.two_byte_,ks=4,vs=2,off=1) R2_w=1") +__naked void stack_load_preserves_const_precision(void) +{ + asm volatile ( + /* establish checkpoint with state that has no stack slots; + * if we bubble up to this state without finding desired stack + * slot, then it's a bug and should be caught + */ + "goto +0;" + + /* fp-8 is const 1 *fake* register */ + ".8byte %[fp8_st_one];" /* LLVM-18+: *(u64 *)(r10 -8) = 1; */ + + /* fp-16 is const 1 register */ + "r0 = 1;" + "*(u64 *)(r10 -16) = r0;" + + /* force checkpoint to check precision marks preserved in parent states */ + "goto +0;" + + /* load single U64 from aligned FAKE_REG=1 slot */ + "r1 = %[two_byte_buf];" + "r2 = *(u64 *)(r10 -8);" + "r1 += r2;" + "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ + + /* load single U64 from aligned REG=1 slot */ + "r1 = %[two_byte_buf];" + "r2 = *(u64 *)(r10 -16);" + "r1 += r2;" + "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ + + "r0 = 0;" + "exit;" + : + : __imm_ptr(two_byte_buf), + __imm_insn(fp8_st_one, BPF_ST_MEM(BPF_DW, BPF_REG_FP, -8, 1)) + : __clobber_common); +} + +SEC("raw_tp") +__log_level(2) __flag(BPF_F_TEST_STATE_FREQ) +__success +/* make sure fp-8 is 32-bit FAKE subregister spill */ +__msg("3: (62) *(u32 *)(r10 -8) = 1 ; R10=fp0 fp-8=????1") +/* but fp-16 is spilled IMPRECISE zero const reg */ +__msg("5: (63) *(u32 *)(r10 -16) = r0 ; R0_w=1 R10=fp0 fp-16=????1") +/* validate load from fp-8, which was initialized using BPF_ST_MEM */ +__msg("8: (61) r2 = *(u32 *)(r10 -8) ; R2_w=1 R10=fp0 fp-8=????1") +__msg("9: (0f) r1 += r2") +__msg("mark_precise: frame0: last_idx 9 first_idx 7 subseq_idx -1") +__msg("mark_precise: frame0: regs=r2 stack= before 8: (61) r2 = *(u32 *)(r10 -8)") +__msg("mark_precise: frame0: regs= stack=-8 before 7: (bf) r1 = r6") +__msg("mark_precise: frame0: parent state regs= stack=-8: R0_w=1 R1=ctx() R6_r=map_value(map=.data.two_byte_,ks=4,vs=2) R10=fp0 fp-8_r=????P1 fp-16=????1") +__msg("mark_precise: frame0: last_idx 6 first_idx 3 subseq_idx 7") +__msg("mark_precise: frame0: regs= stack=-8 before 6: (05) goto pc+0") +__msg("mark_precise: frame0: regs= stack=-8 before 5: (63) *(u32 *)(r10 -16) = r0") +__msg("mark_precise: frame0: regs= stack=-8 before 4: (b7) r0 = 1") +__msg("mark_precise: frame0: regs= stack=-8 before 3: (62) *(u32 *)(r10 -8) = 1") +__msg("10: R1_w=map_value(map=.data.two_byte_,ks=4,vs=2,off=1) R2_w=1") +/* validate load from fp-16, which was initialized using BPF_STX_MEM */ +__msg("12: (61) r2 = *(u32 *)(r10 -16) ; R2_w=1 R10=fp0 fp-16=????1") +__msg("13: (0f) r1 += r2") +__msg("mark_precise: frame0: last_idx 13 first_idx 7 subseq_idx -1") +__msg("mark_precise: frame0: regs=r2 stack= before 12: (61) r2 = *(u32 *)(r10 -16)") +__msg("mark_precise: frame0: regs= stack=-16 before 11: (bf) r1 = r6") +__msg("mark_precise: frame0: regs= stack=-16 before 10: (73) *(u8 *)(r1 +0) = r2") +__msg("mark_precise: frame0: regs= stack=-16 before 9: (0f) r1 += r2") +__msg("mark_precise: frame0: regs= stack=-16 before 8: (61) r2 = *(u32 *)(r10 -8)") +__msg("mark_precise: frame0: regs= stack=-16 before 7: (bf) r1 = r6") +__msg("mark_precise: frame0: parent state regs= stack=-16: R0_w=1 R1=ctx() R6_r=map_value(map=.data.two_byte_,ks=4,vs=2) R10=fp0 fp-8_r=????P1 fp-16_r=????P1") +__msg("mark_precise: frame0: last_idx 6 first_idx 3 subseq_idx 7") +__msg("mark_precise: frame0: regs= stack=-16 before 6: (05) goto pc+0") +__msg("mark_precise: frame0: regs= stack=-16 before 5: (63) *(u32 *)(r10 -16) = r0") +__msg("mark_precise: frame0: regs=r0 stack= before 4: (b7) r0 = 1") +__msg("14: R1_w=map_value(map=.data.two_byte_,ks=4,vs=2,off=1) R2_w=1") +__naked void stack_load_preserves_const_precision_subreg(void) +{ + asm volatile ( + /* establish checkpoint with state that has no stack slots; + * if we bubble up to this state without finding desired stack + * slot, then it's a bug and should be caught + */ + "goto +0;" + + /* fp-8 is const 1 *fake* SUB-register */ + ".8byte %[fp8_st_one];" /* LLVM-18+: *(u32 *)(r10 -8) = 1; */ + + /* fp-16 is const 1 SUB-register */ + "r0 = 1;" + "*(u32 *)(r10 -16) = r0;" + + /* force checkpoint to check precision marks preserved in parent states */ + "goto +0;" + + /* load single U32 from aligned FAKE_REG=1 slot */ + "r1 = %[two_byte_buf];" + "r2 = *(u32 *)(r10 -8);" + "r1 += r2;" + "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ + + /* load single U32 from aligned REG=1 slot */ + "r1 = %[two_byte_buf];" + "r2 = *(u32 *)(r10 -16);" + "r1 += r2;" + "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ + + "r0 = 0;" + "exit;" + : + : __imm_ptr(two_byte_buf), + __imm_insn(fp8_st_one, BPF_ST_MEM(BPF_W, BPF_REG_FP, -8, 1)) /* 32-bit spill */ + : __clobber_common); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c b/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c index e0f77e3e78692a..417c61cd4b1901 100644 --- a/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c +++ b/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c @@ -37,7 +37,7 @@ __naked void ptr_to_stack_store_load(void) SEC("socket") __description("PTR_TO_STACK store/load - bad alignment on off") -__failure __msg("misaligned stack access off (0x0; 0x0)+-8+2 size 8") +__failure __msg("misaligned stack access off 0+-8+2 size 8") __failure_unpriv __naked void load_bad_alignment_on_off(void) { @@ -53,7 +53,7 @@ __naked void load_bad_alignment_on_off(void) SEC("socket") __description("PTR_TO_STACK store/load - bad alignment on reg") -__failure __msg("misaligned stack access off (0x0; 0x0)+-10+8 size 8") +__failure __msg("misaligned stack access off 0+-10+8 size 8") __failure_unpriv __naked void load_bad_alignment_on_reg(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_subprog_precision.c b/tools/testing/selftests/bpf/progs/verifier_subprog_precision.c index b5efcaeaa1ae97..6f5d19665cf67d 100644 --- a/tools/testing/selftests/bpf/progs/verifier_subprog_precision.c +++ b/tools/testing/selftests/bpf/progs/verifier_subprog_precision.c @@ -117,6 +117,56 @@ __naked int global_subprog_result_precise(void) ); } +__naked __noinline __used +static unsigned long loop_callback_bad() +{ + /* bpf_loop() callback that can return values outside of [0, 1] range */ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "if r0 s> 1000 goto 1f;" + "r0 = 0;" + "1:" + "goto +0;" /* checkpoint */ + /* bpf_loop() expects [0, 1] values, so branch above skipping + * r0 = 0; should lead to a failure, but if exit instruction + * doesn't enforce r0's precision, this callback will be + * successfully verified + */ + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_common + ); +} + +SEC("?raw_tp") +__failure __log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +/* check that fallthrough code path marks r0 as precise */ +__msg("mark_precise: frame1: regs=r0 stack= before 11: (b7) r0 = 0") +/* check that we have branch code path doing its own validation */ +__msg("from 10 to 12: frame1: R0=scalar(smin=umin=1001") +/* check that branch code path marks r0 as precise, before failing */ +__msg("mark_precise: frame1: regs=r0 stack= before 9: (85) call bpf_get_prandom_u32#7") +__msg("At callback return the register R0 has smin=1001 should have been in [0, 1]") +__naked int callback_precise_return_fail(void) +{ + asm volatile ( + "r1 = 1;" /* nr_loops */ + "r2 = %[loop_callback_bad];" /* callback_fn */ + "r3 = 0;" /* callback_ctx */ + "r4 = 0;" /* flags */ + "call %[bpf_loop];" + + "r0 = 0;" + "exit;" + : + : __imm_ptr(loop_callback_bad), + __imm(bpf_loop) + : __clobber_common + ); +} + SEC("?raw_tp") __success __log_level(2) /* First simulated path does not include callback body, @@ -539,11 +589,24 @@ static __u64 subprog_spill_reg_precise(void) SEC("?raw_tp") __success __log_level(2) -/* precision backtracking can't currently handle stack access not through r10, - * so we won't be able to mark stack slot fp-8 as precise, and so will - * fallback to forcing all as precise - */ -__msg("mark_precise: frame0: falling back to forcing all scalars precise") +__msg("10: (0f) r1 += r7") +__msg("mark_precise: frame0: last_idx 10 first_idx 7 subseq_idx -1") +__msg("mark_precise: frame0: regs=r7 stack= before 9: (bf) r1 = r8") +__msg("mark_precise: frame0: regs=r7 stack= before 8: (27) r7 *= 4") +__msg("mark_precise: frame0: regs=r7 stack= before 7: (79) r7 = *(u64 *)(r10 -8)") +__msg("mark_precise: frame0: parent state regs= stack=-8: R0_w=2 R6_w=1 R8_rw=map_value(map=.data.vals,ks=4,vs=16) R10=fp0 fp-8_rw=P1") +__msg("mark_precise: frame0: last_idx 18 first_idx 0 subseq_idx 7") +__msg("mark_precise: frame0: regs= stack=-8 before 18: (95) exit") +__msg("mark_precise: frame1: regs= stack= before 17: (0f) r0 += r2") +__msg("mark_precise: frame1: regs= stack= before 16: (79) r2 = *(u64 *)(r1 +0)") +__msg("mark_precise: frame1: regs= stack= before 15: (79) r0 = *(u64 *)(r10 -16)") +__msg("mark_precise: frame1: regs= stack= before 14: (7b) *(u64 *)(r10 -16) = r2") +__msg("mark_precise: frame1: regs= stack= before 13: (7b) *(u64 *)(r1 +0) = r2") +__msg("mark_precise: frame1: regs=r2 stack= before 6: (85) call pc+6") +__msg("mark_precise: frame0: regs=r2 stack= before 5: (bf) r2 = r6") +__msg("mark_precise: frame0: regs=r6 stack= before 4: (07) r1 += -8") +__msg("mark_precise: frame0: regs=r6 stack= before 3: (bf) r1 = r10") +__msg("mark_precise: frame0: regs=r6 stack= before 2: (b7) r6 = 1") __naked int subprog_spill_into_parent_stack_slot_precise(void) { asm volatile ( @@ -578,14 +641,68 @@ __naked int subprog_spill_into_parent_stack_slot_precise(void) ); } -__naked __noinline __used -static __u64 subprog_with_checkpoint(void) +SEC("?raw_tp") +__success __log_level(2) +__msg("17: (0f) r1 += r0") +__msg("mark_precise: frame0: last_idx 17 first_idx 0 subseq_idx -1") +__msg("mark_precise: frame0: regs=r0 stack= before 16: (bf) r1 = r7") +__msg("mark_precise: frame0: regs=r0 stack= before 15: (27) r0 *= 4") +__msg("mark_precise: frame0: regs=r0 stack= before 14: (79) r0 = *(u64 *)(r10 -16)") +__msg("mark_precise: frame0: regs= stack=-16 before 13: (7b) *(u64 *)(r7 -8) = r0") +__msg("mark_precise: frame0: regs=r0 stack= before 12: (79) r0 = *(u64 *)(r8 +16)") +__msg("mark_precise: frame0: regs= stack=-16 before 11: (7b) *(u64 *)(r8 +16) = r0") +__msg("mark_precise: frame0: regs=r0 stack= before 10: (79) r0 = *(u64 *)(r7 -8)") +__msg("mark_precise: frame0: regs= stack=-16 before 9: (7b) *(u64 *)(r10 -16) = r0") +__msg("mark_precise: frame0: regs=r0 stack= before 8: (07) r8 += -32") +__msg("mark_precise: frame0: regs=r0 stack= before 7: (bf) r8 = r10") +__msg("mark_precise: frame0: regs=r0 stack= before 6: (07) r7 += -8") +__msg("mark_precise: frame0: regs=r0 stack= before 5: (bf) r7 = r10") +__msg("mark_precise: frame0: regs=r0 stack= before 21: (95) exit") +__msg("mark_precise: frame1: regs=r0 stack= before 20: (bf) r0 = r1") +__msg("mark_precise: frame1: regs=r1 stack= before 4: (85) call pc+15") +__msg("mark_precise: frame0: regs=r1 stack= before 3: (bf) r1 = r6") +__msg("mark_precise: frame0: regs=r6 stack= before 2: (b7) r6 = 1") +__naked int stack_slot_aliases_precision(void) { asm volatile ( - "r0 = 0;" - /* guaranteed checkpoint if BPF_F_TEST_STATE_FREQ is used */ - "goto +0;" + "r6 = 1;" + /* pass r6 through r1 into subprog to get it back as r0; + * this whole chain will have to be marked as precise later + */ + "r1 = r6;" + "call identity_subprog;" + /* let's setup two registers that are aliased to r10 */ + "r7 = r10;" + "r7 += -8;" /* r7 = r10 - 8 */ + "r8 = r10;" + "r8 += -32;" /* r8 = r10 - 32 */ + /* now spill subprog's return value (a r6 -> r1 -> r0 chain) + * a few times through different stack pointer regs, making + * sure to use r10, r7, and r8 both in LDX and STX insns, and + * *importantly* also using a combination of const var_off and + * insn->off to validate that we record final stack slot + * correctly, instead of relying on just insn->off derivation, + * which is only valid for r10-based stack offset + */ + "*(u64 *)(r10 - 16) = r0;" + "r0 = *(u64 *)(r7 - 8);" /* r7 - 8 == r10 - 16 */ + "*(u64 *)(r8 + 16) = r0;" /* r8 + 16 = r10 - 16 */ + "r0 = *(u64 *)(r8 + 16);" + "*(u64 *)(r7 - 8) = r0;" + "r0 = *(u64 *)(r10 - 16);" + /* get ready to use r0 as an index into array to force precision */ + "r0 *= 4;" + "r1 = %[vals];" + /* here r0->r1->r6 chain is forced to be precise and has to be + * propagated back to the beginning, including through the + * subprog call and all the stack spills and loads + */ + "r1 += r0;" + "r0 = *(u32 *)(r1 + 0);" "exit;" + : + : __imm_ptr(vals) + : __clobber_common, "r6" ); } diff --git a/tools/testing/selftests/bpf/progs/verifier_var_off.c b/tools/testing/selftests/bpf/progs/verifier_var_off.c index 83a90afba78576..c810f4f6f479c6 100644 --- a/tools/testing/selftests/bpf/progs/verifier_var_off.c +++ b/tools/testing/selftests/bpf/progs/verifier_var_off.c @@ -59,9 +59,10 @@ __naked void stack_read_priv_vs_unpriv(void) " ::: __clobber_all); } -SEC("lwt_in") +SEC("cgroup/skb") __description("variable-offset stack read, uninitialized") -__failure __msg("invalid variable-offset read from stack R2") +__success +__failure_unpriv __msg_unpriv("R2 variable stack access prohibited for !root") __naked void variable_offset_stack_read_uninitialized(void) { asm volatile (" \ @@ -83,12 +84,55 @@ __naked void variable_offset_stack_read_uninitialized(void) SEC("socket") __description("variable-offset stack write, priv vs unpriv") -__success __failure_unpriv +__success +/* Check that the maximum stack depth is correctly maintained according to the + * maximum possible variable offset. + */ +__log_level(4) __msg("stack depth 16") +__failure_unpriv /* Variable stack access is rejected for unprivileged. */ __msg_unpriv("R2 variable stack access prohibited for !root") __retval(0) __naked void stack_write_priv_vs_unpriv(void) +{ + asm volatile (" \ + /* Get an unknown value */ \ + r2 = *(u32*)(r1 + 0); \ + /* Make it small and 8-byte aligned */ \ + r2 &= 8; \ + r2 -= 16; \ + /* Add it to fp. We now have either fp-8 or \ + * fp-16, but we don't know which \ + */ \ + r2 += r10; \ + /* Dereference it for a stack write */ \ + r0 = 0; \ + *(u64*)(r2 + 0) = r0; \ + exit; \ +" ::: __clobber_all); +} + +/* Similar to the previous test, but this time also perform a read from the + * address written to with a variable offset. The read is allowed, showing that, + * after a variable-offset write, a priviledged program can read the slots that + * were in the range of that write (even if the verifier doesn't actually know if + * the slot being read was really written to or not. + * + * Despite this test being mostly a superset, the previous test is also kept for + * the sake of it checking the stack depth in the case where there is no read. + */ +SEC("socket") +__description("variable-offset stack write followed by read") +__success +/* Check that the maximum stack depth is correctly maintained according to the + * maximum possible variable offset. + */ +__log_level(4) __msg("stack depth 16") +__failure_unpriv +__msg_unpriv("R2 variable stack access prohibited for !root") +__retval(0) +__naked void stack_write_followed_by_read(void) { asm volatile (" \ /* Get an unknown value */ \ @@ -103,12 +147,7 @@ __naked void stack_write_priv_vs_unpriv(void) /* Dereference it for a stack write */ \ r0 = 0; \ *(u64*)(r2 + 0) = r0; \ - /* Now read from the address we just wrote. This shows\ - * that, after a variable-offset write, a priviledged\ - * program can read the slots that were in the range of\ - * that write (even if the verifier doesn't actually know\ - * if the slot being read was really written to or not.\ - */ \ + /* Now read from the address we just wrote. */ \ r3 = *(u64*)(r2 + 0); \ r0 = 0; \ exit; \ @@ -224,6 +263,35 @@ __naked void access_max_out_of_bound(void) : __clobber_all); } +/* Similar to the test above, but this time check the special case of a + * zero-sized stack access. We used to have a bug causing crashes for zero-sized + * out-of-bounds accesses. + */ +SEC("socket") +__description("indirect variable-offset stack access, zero-sized, max out of bound") +__failure __msg("invalid variable-offset indirect access to stack R1") +__naked void zero_sized_access_max_out_of_bound(void) +{ + asm volatile (" \ + r0 = 0; \ + /* Fill some stack */ \ + *(u64*)(r10 - 16) = r0; \ + *(u64*)(r10 - 8) = r0; \ + /* Get an unknown value */ \ + r1 = *(u32*)(r1 + 0); \ + r1 &= 63; \ + r1 += -16; \ + /* r1 is now anywhere in [-16,48) */ \ + r1 += r10; \ + r2 = 0; \ + r3 = 0; \ + call %[bpf_probe_read_kernel]; \ + exit; \ +" : + : __imm(bpf_probe_read_kernel) + : __clobber_all); +} + SEC("lwt_in") __description("indirect variable-offset stack access, min out of bound") __failure __msg("invalid variable-offset indirect access to stack R2") @@ -253,9 +321,10 @@ __naked void access_min_out_of_bound(void) : __clobber_all); } -SEC("lwt_in") +SEC("cgroup/skb") __description("indirect variable-offset stack access, min_off < min_initialized") -__failure __msg("invalid indirect read from stack R2 var_off") +__success +__failure_unpriv __msg_unpriv("R2 variable stack access prohibited for !root") __naked void access_min_off_min_initialized(void) { asm volatile (" \ diff --git a/tools/testing/selftests/bpf/progs/xdp_hw_metadata.c b/tools/testing/selftests/bpf/progs/xdp_hw_metadata.c index f6d1cc9ad89299..330ece2eabdb45 100644 --- a/tools/testing/selftests/bpf/progs/xdp_hw_metadata.c +++ b/tools/testing/selftests/bpf/progs/xdp_hw_metadata.c @@ -20,21 +20,32 @@ extern int bpf_xdp_metadata_rx_timestamp(const struct xdp_md *ctx, __u64 *timestamp) __ksym; extern int bpf_xdp_metadata_rx_hash(const struct xdp_md *ctx, __u32 *hash, enum xdp_rss_hash_type *rss_type) __ksym; +extern int bpf_xdp_metadata_rx_vlan_tag(const struct xdp_md *ctx, + __be16 *vlan_proto, + __u16 *vlan_tci) __ksym; SEC("xdp.frags") int rx(struct xdp_md *ctx) { void *data, *data_meta, *data_end; struct ipv6hdr *ip6h = NULL; - struct ethhdr *eth = NULL; struct udphdr *udp = NULL; struct iphdr *iph = NULL; struct xdp_meta *meta; + struct ethhdr *eth; int err; data = (void *)(long)ctx->data; data_end = (void *)(long)ctx->data_end; eth = data; + + if (eth + 1 < data_end && (eth->h_proto == bpf_htons(ETH_P_8021AD) || + eth->h_proto == bpf_htons(ETH_P_8021Q))) + eth = (void *)eth + sizeof(struct vlan_hdr); + + if (eth + 1 < data_end && eth->h_proto == bpf_htons(ETH_P_8021Q)) + eth = (void *)eth + sizeof(struct vlan_hdr); + if (eth + 1 < data_end) { if (eth->h_proto == bpf_htons(ETH_P_IP)) { iph = (void *)(eth + 1); @@ -76,15 +87,28 @@ int rx(struct xdp_md *ctx) return XDP_PASS; } + meta->hint_valid = 0; + + meta->xdp_timestamp = bpf_ktime_get_tai_ns(); err = bpf_xdp_metadata_rx_timestamp(ctx, &meta->rx_timestamp); - if (!err) - meta->xdp_timestamp = bpf_ktime_get_tai_ns(); + if (err) + meta->rx_timestamp_err = err; else - meta->rx_timestamp = 0; /* Used by AF_XDP as not avail signal */ + meta->hint_valid |= XDP_META_FIELD_TS; - err = bpf_xdp_metadata_rx_hash(ctx, &meta->rx_hash, &meta->rx_hash_type); - if (err < 0) - meta->rx_hash_err = err; /* Used by AF_XDP as no hash signal */ + err = bpf_xdp_metadata_rx_hash(ctx, &meta->rx_hash, + &meta->rx_hash_type); + if (err) + meta->rx_hash_err = err; + else + meta->hint_valid |= XDP_META_FIELD_RSS; + + err = bpf_xdp_metadata_rx_vlan_tag(ctx, &meta->rx_vlan_proto, + &meta->rx_vlan_tci); + if (err) + meta->rx_vlan_tag_err = err; + else + meta->hint_valid |= XDP_META_FIELD_VLAN_TAG; __sync_add_and_fetch(&pkts_redir, 1); return bpf_redirect_map(&xsk, ctx->rx_queue_index, XDP_PASS); diff --git a/tools/testing/selftests/bpf/progs/xdp_metadata.c b/tools/testing/selftests/bpf/progs/xdp_metadata.c index d151d406a123ef..31ca229bb3c0f1 100644 --- a/tools/testing/selftests/bpf/progs/xdp_metadata.c +++ b/tools/testing/selftests/bpf/progs/xdp_metadata.c @@ -23,15 +23,47 @@ extern int bpf_xdp_metadata_rx_timestamp(const struct xdp_md *ctx, __u64 *timestamp) __ksym; extern int bpf_xdp_metadata_rx_hash(const struct xdp_md *ctx, __u32 *hash, enum xdp_rss_hash_type *rss_type) __ksym; +extern int bpf_xdp_metadata_rx_vlan_tag(const struct xdp_md *ctx, + __be16 *vlan_proto, + __u16 *vlan_tci) __ksym; SEC("xdp") int rx(struct xdp_md *ctx) { - void *data, *data_meta; + void *data, *data_meta, *data_end; + struct ipv6hdr *ip6h = NULL; + struct ethhdr *eth = NULL; + struct udphdr *udp = NULL; + struct iphdr *iph = NULL; struct xdp_meta *meta; u64 timestamp = -1; int ret; + data = (void *)(long)ctx->data; + data_end = (void *)(long)ctx->data_end; + eth = data; + if (eth + 1 < data_end) { + if (eth->h_proto == bpf_htons(ETH_P_IP)) { + iph = (void *)(eth + 1); + if (iph + 1 < data_end && iph->protocol == IPPROTO_UDP) + udp = (void *)(iph + 1); + } + if (eth->h_proto == bpf_htons(ETH_P_IPV6)) { + ip6h = (void *)(eth + 1); + if (ip6h + 1 < data_end && ip6h->nexthdr == IPPROTO_UDP) + udp = (void *)(ip6h + 1); + } + if (udp && udp + 1 > data_end) + udp = NULL; + } + + if (!udp) + return XDP_PASS; + + /* Forwarding UDP:8080 to AF_XDP */ + if (udp->dest != bpf_htons(8080)) + return XDP_PASS; + /* Reserve enough for all custom metadata. */ ret = bpf_xdp_adjust_meta(ctx, -(int)sizeof(struct xdp_meta)); @@ -57,6 +89,8 @@ int rx(struct xdp_md *ctx) meta->rx_timestamp = 1; bpf_xdp_metadata_rx_hash(ctx, &meta->rx_hash, &meta->rx_hash_type); + bpf_xdp_metadata_rx_vlan_tag(ctx, &meta->rx_vlan_proto, + &meta->rx_vlan_tci); return bpf_redirect_map(&xsk, ctx->rx_queue_index, XDP_PASS); } diff --git a/tools/testing/selftests/bpf/progs/xdp_synproxy_kern.c b/tools/testing/selftests/bpf/progs/xdp_synproxy_kern.c index 80f620602d50ff..518329c666e93f 100644 --- a/tools/testing/selftests/bpf/progs/xdp_synproxy_kern.c +++ b/tools/testing/selftests/bpf/progs/xdp_synproxy_kern.c @@ -467,13 +467,13 @@ static __always_inline int tcp_lookup(void *ctx, struct header_pointers *hdr, bo unsigned long status = ct->status; bpf_ct_release(ct); - if (status & IPS_CONFIRMED_BIT) + if (status & IPS_CONFIRMED) return XDP_PASS; } else if (ct_lookup_opts.error != -ENOENT) { return XDP_ABORTED; } - /* error == -ENOENT || !(status & IPS_CONFIRMED_BIT) */ + /* error == -ENOENT || !(status & IPS_CONFIRMED) */ return XDP_TX; } diff --git a/tools/testing/selftests/bpf/test_loader.c b/tools/testing/selftests/bpf/test_loader.c index a350ecdfba4a82..74ceb7877ae220 100644 --- a/tools/testing/selftests/bpf/test_loader.c +++ b/tools/testing/selftests/bpf/test_loader.c @@ -27,6 +27,7 @@ #define TEST_TAG_RETVAL_PFX_UNPRIV "comment:test_retval_unpriv=" #define TEST_TAG_AUXILIARY "comment:test_auxiliary" #define TEST_TAG_AUXILIARY_UNPRIV "comment:test_auxiliary_unpriv" +#define TEST_BTF_PATH "comment:test_btf_path=" /* Warning: duplicated in bpf_misc.h */ #define POINTER_VALUE 0xcafe4all @@ -58,6 +59,7 @@ struct test_spec { const char *prog_name; struct test_subspec priv; struct test_subspec unpriv; + const char *btf_custom_path; int log_level; int prog_flags; int mode_mask; @@ -288,6 +290,8 @@ static int parse_test_spec(struct test_loader *tester, goto cleanup; update_flags(&spec->prog_flags, flags, clear); } + } else if (str_has_pfx(s, TEST_BTF_PATH)) { + spec->btf_custom_path = s + sizeof(TEST_BTF_PATH) - 1; } } @@ -578,6 +582,9 @@ void run_subtest(struct test_loader *tester, } } + /* Implicitly reset to NULL if next test case doesn't specify */ + open_opts->btf_custom_path = spec->btf_custom_path; + tobj = bpf_object__open_mem(obj_bytes, obj_byte_cnt, open_opts); if (!ASSERT_OK_PTR(tobj, "obj_open_mem")) /* shouldn't happen */ goto subtest_cleanup; diff --git a/tools/testing/selftests/bpf/test_tunnel.sh b/tools/testing/selftests/bpf/test_tunnel.sh index 2dec7dbf29a27b..d9661b9988ba90 100755 --- a/tools/testing/selftests/bpf/test_tunnel.sh +++ b/tools/testing/selftests/bpf/test_tunnel.sh @@ -517,90 +517,6 @@ test_ip6ip6() echo -e ${GREEN}"PASS: ip6$TYPE"${NC} } -setup_xfrm_tunnel() -{ - auth=0x$(printf '1%.0s' {1..40}) - enc=0x$(printf '2%.0s' {1..32}) - spi_in_to_out=0x1 - spi_out_to_in=0x2 - # at_ns0 namespace - # at_ns0 -> root - ip netns exec at_ns0 \ - ip xfrm state add src 172.16.1.100 dst 172.16.1.200 proto esp \ - spi $spi_in_to_out reqid 1 mode tunnel \ - auth-trunc 'hmac(sha1)' $auth 96 enc 'cbc(aes)' $enc - ip netns exec at_ns0 \ - ip xfrm policy add src 10.1.1.100/32 dst 10.1.1.200/32 dir out \ - tmpl src 172.16.1.100 dst 172.16.1.200 proto esp reqid 1 \ - mode tunnel - # root -> at_ns0 - ip netns exec at_ns0 \ - ip xfrm state add src 172.16.1.200 dst 172.16.1.100 proto esp \ - spi $spi_out_to_in reqid 2 mode tunnel \ - auth-trunc 'hmac(sha1)' $auth 96 enc 'cbc(aes)' $enc - ip netns exec at_ns0 \ - ip xfrm policy add src 10.1.1.200/32 dst 10.1.1.100/32 dir in \ - tmpl src 172.16.1.200 dst 172.16.1.100 proto esp reqid 2 \ - mode tunnel - # address & route - ip netns exec at_ns0 \ - ip addr add dev veth0 10.1.1.100/32 - ip netns exec at_ns0 \ - ip route add 10.1.1.200 dev veth0 via 172.16.1.200 \ - src 10.1.1.100 - - # root namespace - # at_ns0 -> root - ip xfrm state add src 172.16.1.100 dst 172.16.1.200 proto esp \ - spi $spi_in_to_out reqid 1 mode tunnel \ - auth-trunc 'hmac(sha1)' $auth 96 enc 'cbc(aes)' $enc - ip xfrm policy add src 10.1.1.100/32 dst 10.1.1.200/32 dir in \ - tmpl src 172.16.1.100 dst 172.16.1.200 proto esp reqid 1 \ - mode tunnel - # root -> at_ns0 - ip xfrm state add src 172.16.1.200 dst 172.16.1.100 proto esp \ - spi $spi_out_to_in reqid 2 mode tunnel \ - auth-trunc 'hmac(sha1)' $auth 96 enc 'cbc(aes)' $enc - ip xfrm policy add src 10.1.1.200/32 dst 10.1.1.100/32 dir out \ - tmpl src 172.16.1.200 dst 172.16.1.100 proto esp reqid 2 \ - mode tunnel - # address & route - ip addr add dev veth1 10.1.1.200/32 - ip route add 10.1.1.100 dev veth1 via 172.16.1.100 src 10.1.1.200 -} - -test_xfrm_tunnel() -{ - if [[ -e /sys/kernel/tracing/trace ]]; then - TRACE=/sys/kernel/tracing/trace - else - TRACE=/sys/kernel/debug/tracing/trace - fi - config_device - > ${TRACE} - setup_xfrm_tunnel - mkdir -p ${BPF_PIN_TUNNEL_DIR} - bpftool prog loadall ${BPF_FILE} ${BPF_PIN_TUNNEL_DIR} - tc qdisc add dev veth1 clsact - tc filter add dev veth1 proto ip ingress bpf da object-pinned \ - ${BPF_PIN_TUNNEL_DIR}/xfrm_get_state - ip netns exec at_ns0 ping $PING_ARG 10.1.1.200 - sleep 1 - grep "reqid 1" ${TRACE} - check_err $? - grep "spi 0x1" ${TRACE} - check_err $? - grep "remote ip 0xac100164" ${TRACE} - check_err $? - cleanup - - if [ $ret -ne 0 ]; then - echo -e ${RED}"FAIL: xfrm tunnel"${NC} - return 1 - fi - echo -e ${GREEN}"PASS: xfrm tunnel"${NC} -} - attach_bpf() { DEV=$1 @@ -630,10 +546,6 @@ cleanup() ip link del ip6geneve11 2> /dev/null ip link del erspan11 2> /dev/null ip link del ip6erspan11 2> /dev/null - ip xfrm policy delete dir out src 10.1.1.200/32 dst 10.1.1.100/32 2> /dev/null - ip xfrm policy delete dir in src 10.1.1.100/32 dst 10.1.1.200/32 2> /dev/null - ip xfrm state delete src 172.16.1.100 dst 172.16.1.200 proto esp spi 0x1 2> /dev/null - ip xfrm state delete src 172.16.1.200 dst 172.16.1.100 proto esp spi 0x2 2> /dev/null } cleanup_exit() @@ -716,10 +628,6 @@ bpf_tunnel_test() test_ip6ip6 errors=$(( $errors + $? )) - echo "Testing IPSec tunnel..." - test_xfrm_tunnel - errors=$(( $errors + $? )) - return $errors } diff --git a/tools/testing/selftests/bpf/testing_helpers.h b/tools/testing/selftests/bpf/testing_helpers.h index 5b7a55136741ac..35284faff4f293 100644 --- a/tools/testing/selftests/bpf/testing_helpers.h +++ b/tools/testing/selftests/bpf/testing_helpers.h @@ -9,6 +9,9 @@ #include #include +#define __TO_STR(x) #x +#define TO_STR(x) __TO_STR(x) + int parse_num_list(const char *s, bool **set, int *set_len); __u32 link_info_prog_id(const struct bpf_link *link, struct bpf_link_info *info); int bpf_prog_test_load(const char *file, enum bpf_prog_type type, diff --git a/tools/testing/selftests/bpf/verifier/atomic_cmpxchg.c b/tools/testing/selftests/bpf/verifier/atomic_cmpxchg.c index 319337bdcfc856..9a7b1106fda812 100644 --- a/tools/testing/selftests/bpf/verifier/atomic_cmpxchg.c +++ b/tools/testing/selftests/bpf/verifier/atomic_cmpxchg.c @@ -83,17 +83,6 @@ .result = REJECT, .errstr = "!read_ok", }, -{ - "Can't use cmpxchg on uninit memory", - .insns = { - BPF_MOV64_IMM(BPF_REG_0, 3), - BPF_MOV64_IMM(BPF_REG_2, 4), - BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_10, BPF_REG_2, -8), - BPF_EXIT_INSN(), - }, - .result = REJECT, - .errstr = "invalid read from stack", -}, { "BPF_W cmpxchg should zero top 32 bits", .insns = { diff --git a/tools/testing/selftests/bpf/verifier/calls.c b/tools/testing/selftests/bpf/verifier/calls.c index 3d5cd51071f047..ab25a81fd3a108 100644 --- a/tools/testing/selftests/bpf/verifier/calls.c +++ b/tools/testing/selftests/bpf/verifier/calls.c @@ -1505,7 +1505,9 @@ .prog_type = BPF_PROG_TYPE_XDP, .fixup_map_hash_8b = { 23 }, .result = REJECT, - .errstr = "invalid read from stack R7 off=-16 size=8", + .errstr = "R0 invalid mem access 'scalar'", + .result_unpriv = REJECT, + .errstr_unpriv = "invalid read from stack R7 off=-16 size=8", }, { "calls: two calls that receive map_value via arg=ptr_stack_of_caller. test1", diff --git a/tools/testing/selftests/bpf/verifier/precise.c b/tools/testing/selftests/bpf/verifier/precise.c index 0d84dd1f38b6b0..8a2ff81d835088 100644 --- a/tools/testing/selftests/bpf/verifier/precise.c +++ b/tools/testing/selftests/bpf/verifier/precise.c @@ -140,10 +140,11 @@ .result = REJECT, }, { - "precise: ST insn causing spi > allocated_stack", + "precise: ST zero to stack insn is supported", .insns = { BPF_MOV64_REG(BPF_REG_3, BPF_REG_10), BPF_JMP_IMM(BPF_JNE, BPF_REG_3, 123, 0), + /* not a register spill, so we stop precision propagation for R4 here */ BPF_ST_MEM(BPF_DW, BPF_REG_3, -8, 0), BPF_LDX_MEM(BPF_DW, BPF_REG_4, BPF_REG_10, -8), BPF_MOV64_IMM(BPF_REG_0, -1), @@ -157,11 +158,11 @@ mark_precise: frame0: last_idx 4 first_idx 2\ mark_precise: frame0: regs=r4 stack= before 4\ mark_precise: frame0: regs=r4 stack= before 3\ - mark_precise: frame0: regs= stack=-8 before 2\ - mark_precise: frame0: falling back to forcing all scalars precise\ - force_precise: frame0: forcing r0 to be precise\ mark_precise: frame0: last_idx 5 first_idx 5\ - mark_precise: frame0: parent state regs= stack=:", + mark_precise: frame0: parent state regs=r0 stack=:\ + mark_precise: frame0: last_idx 4 first_idx 2\ + mark_precise: frame0: regs=r0 stack= before 4\ + 5: R0=-1 R4=0", .result = VERBOSE_ACCEPT, .retval = -1, }, @@ -169,6 +170,8 @@ "precise: STX insn causing spi > allocated_stack", .insns = { BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_get_prandom_u32), + /* make later reg spill more interesting by having somewhat known scalar */ + BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xff), BPF_MOV64_REG(BPF_REG_3, BPF_REG_10), BPF_JMP_IMM(BPF_JNE, BPF_REG_3, 123, 0), BPF_STX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, -8), @@ -179,18 +182,21 @@ }, .prog_type = BPF_PROG_TYPE_XDP, .flags = BPF_F_TEST_STATE_FREQ, - .errstr = "mark_precise: frame0: last_idx 6 first_idx 6\ + .errstr = "mark_precise: frame0: last_idx 7 first_idx 7\ mark_precise: frame0: parent state regs=r4 stack=:\ - mark_precise: frame0: last_idx 5 first_idx 3\ - mark_precise: frame0: regs=r4 stack= before 5\ - mark_precise: frame0: regs=r4 stack= before 4\ - mark_precise: frame0: regs= stack=-8 before 3\ - mark_precise: frame0: falling back to forcing all scalars precise\ - force_precise: frame0: forcing r0 to be precise\ - force_precise: frame0: forcing r0 to be precise\ - force_precise: frame0: forcing r0 to be precise\ - force_precise: frame0: forcing r0 to be precise\ - mark_precise: frame0: last_idx 6 first_idx 6\ + mark_precise: frame0: last_idx 6 first_idx 4\ + mark_precise: frame0: regs=r4 stack= before 6: (b7) r0 = -1\ + mark_precise: frame0: regs=r4 stack= before 5: (79) r4 = *(u64 *)(r10 -8)\ + mark_precise: frame0: regs= stack=-8 before 4: (7b) *(u64 *)(r3 -8) = r0\ + mark_precise: frame0: parent state regs=r0 stack=:\ + mark_precise: frame0: last_idx 3 first_idx 3\ + mark_precise: frame0: regs=r0 stack= before 3: (55) if r3 != 0x7b goto pc+0\ + mark_precise: frame0: regs=r0 stack= before 2: (bf) r3 = r10\ + mark_precise: frame0: regs=r0 stack= before 1: (57) r0 &= 255\ + mark_precise: frame0: parent state regs=r0 stack=:\ + mark_precise: frame0: last_idx 0 first_idx 0\ + mark_precise: frame0: regs=r0 stack= before 0: (85) call bpf_get_prandom_u32#7\ + mark_precise: frame0: last_idx 7 first_idx 7\ mark_precise: frame0: parent state regs= stack=:", .result = VERBOSE_ACCEPT, .retval = -1, diff --git a/tools/testing/selftests/bpf/verify_sig_setup.sh b/tools/testing/selftests/bpf/verify_sig_setup.sh index ba08922b4a278d..f2cac42298ba3b 100755 --- a/tools/testing/selftests/bpf/verify_sig_setup.sh +++ b/tools/testing/selftests/bpf/verify_sig_setup.sh @@ -60,6 +60,27 @@ cleanup() { rm -rf ${tmp_dir} } +fsverity_create_sign_file() { + local tmp_dir="$1" + + data_file=${tmp_dir}/data-file + sig_file=${tmp_dir}/sig-file + dd if=/dev/urandom of=$data_file bs=1 count=12345 2> /dev/null + fsverity sign --key ${tmp_dir}/signing_key.pem $data_file $sig_file + + # We do not want to enable fsverity on $data_file yet. Try whether + # the file system support fsverity on a different file. + touch ${tmp_dir}/tmp-file + fsverity enable ${tmp_dir}/tmp-file +} + +fsverity_enable_file() { + local tmp_dir="$1" + + data_file=${tmp_dir}/data-file + fsverity enable $data_file +} + catch() { local exit_code="$1" @@ -86,6 +107,10 @@ main() setup "${tmp_dir}" elif [[ "${action}" == "cleanup" ]]; then cleanup "${tmp_dir}" + elif [[ "${action}" == "fsverity-create-sign" ]]; then + fsverity_create_sign_file "${tmp_dir}" + elif [[ "${action}" == "fsverity-enable" ]]; then + fsverity_enable_file "${tmp_dir}" else echo "Unknown action: ${action}" exit 1 diff --git a/tools/testing/selftests/bpf/veristat.c b/tools/testing/selftests/bpf/veristat.c index 1d418d66e3758a..244d4996e06efe 100644 --- a/tools/testing/selftests/bpf/veristat.c +++ b/tools/testing/selftests/bpf/veristat.c @@ -1254,7 +1254,7 @@ static int cmp_join_stat(const struct verif_stats_join *s1, bool asc, bool abs) { const char *str1 = NULL, *str2 = NULL; - double v1, v2; + double v1 = 0.0, v2 = 0.0; int cmp = 0; fetch_join_stat_value(s1, id, var, &str1, &v1); diff --git a/tools/testing/selftests/bpf/xdp_hw_metadata.c b/tools/testing/selftests/bpf/xdp_hw_metadata.c index 3291625ba4fba9..878d68db032567 100644 --- a/tools/testing/selftests/bpf/xdp_hw_metadata.c +++ b/tools/testing/selftests/bpf/xdp_hw_metadata.c @@ -21,6 +21,9 @@ #include "xsk.h" #include +#include +#include +#include #include #include #include @@ -79,7 +82,7 @@ static int open_xsk(int ifindex, struct xsk *xsk, __u32 queue_id) .flags = XSK_UMEM__DEFAULT_FLAGS, .tx_metadata_len = sizeof(struct xsk_tx_metadata), }; - __u32 idx; + __u32 idx = 0; u64 addr; int ret; int i; @@ -182,19 +185,31 @@ static void print_tstamp_delta(const char *name, const char *refname, (double)delta / 1000); } +#define VLAN_PRIO_MASK GENMASK(15, 13) /* Priority Code Point */ +#define VLAN_DEI_MASK GENMASK(12, 12) /* Drop Eligible Indicator */ +#define VLAN_VID_MASK GENMASK(11, 0) /* VLAN Identifier */ +static void print_vlan_tci(__u16 tag) +{ + __u16 vlan_id = FIELD_GET(VLAN_VID_MASK, tag); + __u8 pcp = FIELD_GET(VLAN_PRIO_MASK, tag); + bool dei = FIELD_GET(VLAN_DEI_MASK, tag); + + printf("PCP=%u, DEI=%d, VID=0x%X\n", pcp, dei, vlan_id); +} + static void verify_xdp_metadata(void *data, clockid_t clock_id) { struct xdp_meta *meta; meta = data - sizeof(*meta); - if (meta->rx_hash_err < 0) - printf("No rx_hash err=%d\n", meta->rx_hash_err); - else + if (meta->hint_valid & XDP_META_FIELD_RSS) printf("rx_hash: 0x%X with RSS type:0x%X\n", meta->rx_hash, meta->rx_hash_type); + else + printf("No rx_hash, err=%d\n", meta->rx_hash_err); - if (meta->rx_timestamp) { + if (meta->hint_valid & XDP_META_FIELD_TS) { __u64 ref_tstamp = gettime(clock_id); /* store received timestamps to calculate a delta at tx */ @@ -206,7 +221,16 @@ static void verify_xdp_metadata(void *data, clockid_t clock_id) print_tstamp_delta("XDP RX-time", "User RX-time", meta->xdp_timestamp, ref_tstamp); } else { - printf("No rx_timestamp\n"); + printf("No rx_timestamp, err=%d\n", meta->rx_timestamp_err); + } + + if (meta->hint_valid & XDP_META_FIELD_VLAN_TAG) { + printf("rx_vlan_proto: 0x%X\n", ntohs(meta->rx_vlan_proto)); + printf("rx_vlan_tci: "); + print_vlan_tci(meta->rx_vlan_tci); + } else { + printf("No rx_vlan_tci or rx_vlan_proto, err=%d\n", + meta->rx_vlan_tag_err); } } diff --git a/tools/testing/selftests/bpf/xdp_metadata.h b/tools/testing/selftests/bpf/xdp_metadata.h index 938a729bd3072f..87318ad1117a1d 100644 --- a/tools/testing/selftests/bpf/xdp_metadata.h +++ b/tools/testing/selftests/bpf/xdp_metadata.h @@ -9,12 +9,44 @@ #define ETH_P_IPV6 0x86DD #endif +#ifndef ETH_P_8021Q +#define ETH_P_8021Q 0x8100 +#endif + +#ifndef ETH_P_8021AD +#define ETH_P_8021AD 0x88A8 +#endif + +#ifndef BIT +#define BIT(nr) (1 << (nr)) +#endif + +/* Non-existent checksum status */ +#define XDP_CHECKSUM_MAGIC BIT(2) + +enum xdp_meta_field { + XDP_META_FIELD_TS = BIT(0), + XDP_META_FIELD_RSS = BIT(1), + XDP_META_FIELD_VLAN_TAG = BIT(2), +}; + struct xdp_meta { - __u64 rx_timestamp; + union { + __u64 rx_timestamp; + __s32 rx_timestamp_err; + }; __u64 xdp_timestamp; __u32 rx_hash; union { __u32 rx_hash_type; __s32 rx_hash_err; }; + union { + struct { + __be16 rx_vlan_proto; + __u16 rx_vlan_tci; + }; + __s32 rx_vlan_tag_err; + }; + enum xdp_meta_field hint_valid; }; diff --git a/tools/testing/selftests/bpf/xskxceiver.c b/tools/testing/selftests/bpf/xskxceiver.c index b604c570309a7f..b1102ee13faa12 100644 --- a/tools/testing/selftests/bpf/xskxceiver.c +++ b/tools/testing/selftests/bpf/xskxceiver.c @@ -634,16 +634,24 @@ static u32 pkt_nb_frags(u32 frame_size, struct pkt_stream *pkt_stream, struct pk return nb_frags; } +static bool set_pkt_valid(int offset, u32 len) +{ + return len <= MAX_ETH_JUMBO_SIZE; +} + static void pkt_set(struct pkt_stream *pkt_stream, struct pkt *pkt, int offset, u32 len) { pkt->offset = offset; pkt->len = len; - if (len > MAX_ETH_JUMBO_SIZE) { - pkt->valid = false; - } else { - pkt->valid = true; - pkt_stream->nb_valid_entries++; - } + pkt->valid = set_pkt_valid(offset, len); +} + +static void pkt_stream_pkt_set(struct pkt_stream *pkt_stream, struct pkt *pkt, int offset, u32 len) +{ + bool prev_pkt_valid = pkt->valid; + + pkt_set(pkt_stream, pkt, offset, len); + pkt_stream->nb_valid_entries += pkt->valid - prev_pkt_valid; } static u32 pkt_get_buffer_len(struct xsk_umem_info *umem, u32 len) @@ -665,7 +673,7 @@ static struct pkt_stream *__pkt_stream_generate(u32 nb_pkts, u32 pkt_len, u32 nb for (i = 0; i < nb_pkts; i++) { struct pkt *pkt = &pkt_stream->pkts[i]; - pkt_set(pkt_stream, pkt, 0, pkt_len); + pkt_stream_pkt_set(pkt_stream, pkt, 0, pkt_len); pkt->pkt_nb = nb_start + i * nb_off; } @@ -700,10 +708,9 @@ static void __pkt_stream_replace_half(struct ifobject *ifobj, u32 pkt_len, pkt_stream = pkt_stream_clone(ifobj->xsk->pkt_stream); for (i = 1; i < ifobj->xsk->pkt_stream->nb_pkts; i += 2) - pkt_set(pkt_stream, &pkt_stream->pkts[i], offset, pkt_len); + pkt_stream_pkt_set(pkt_stream, &pkt_stream->pkts[i], offset, pkt_len); ifobj->xsk->pkt_stream = pkt_stream; - pkt_stream->nb_valid_entries /= 2; } static void pkt_stream_replace_half(struct test_spec *test, u32 pkt_len, int offset) diff --git a/tools/testing/selftests/net/forwarding/Makefile b/tools/testing/selftests/net/forwarding/Makefile index df593b7b3e6bf3..452693514be4b0 100644 --- a/tools/testing/selftests/net/forwarding/Makefile +++ b/tools/testing/selftests/net/forwarding/Makefile @@ -17,6 +17,7 @@ TEST_PROGS = bridge_fdb_learning_limit.sh \ dual_vxlan_bridge.sh \ ethtool_extended_state.sh \ ethtool_mm.sh \ + ethtool_rmon.sh \ ethtool.sh \ gre_custom_multipath_hash.sh \ gre_inner_v4_multipath.sh \ diff --git a/tools/testing/selftests/net/forwarding/ethtool_rmon.sh b/tools/testing/selftests/net/forwarding/ethtool_rmon.sh new file mode 100755 index 00000000000000..41a34a61f7632e --- /dev/null +++ b/tools/testing/selftests/net/forwarding/ethtool_rmon.sh @@ -0,0 +1,143 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +ALL_TESTS=" + rmon_rx_histogram + rmon_tx_histogram +" + +NUM_NETIFS=2 +source lib.sh + +ETH_FCS_LEN=4 +ETH_HLEN=$((6+6+2)) + +declare -A netif_mtu + +ensure_mtu() +{ + local iface=$1; shift + local len=$1; shift + local current=$(ip -j link show dev $iface | jq -r '.[0].mtu') + local required=$((len - ETH_HLEN - ETH_FCS_LEN)) + + if [ $current -lt $required ]; then + ip link set dev $iface mtu $required || return 1 + fi +} + +bucket_test() +{ + local iface=$1; shift + local neigh=$1; shift + local set=$1; shift + local bucket=$1; shift + local len=$1; shift + local num_rx=10000 + local num_tx=20000 + local expected= + local before= + local after= + local delta= + + # Mausezahn does not include FCS bytes in its length - but the + # histogram counters do + len=$((len - ETH_FCS_LEN)) + + before=$(ethtool --json -S $iface --groups rmon | \ + jq -r ".[0].rmon[\"${set}-pktsNtoM\"][$bucket].val") + + # Send 10k one way and 20k in the other, to detect counters + # mapped to the wrong direction + $MZ $neigh -q -c $num_rx -p $len -a own -b bcast -d 10us + $MZ $iface -q -c $num_tx -p $len -a own -b bcast -d 10us + + after=$(ethtool --json -S $iface --groups rmon | \ + jq -r ".[0].rmon[\"${set}-pktsNtoM\"][$bucket].val") + + delta=$((after - before)) + + expected=$([ $set = rx ] && echo $num_rx || echo $num_tx) + + # Allow some extra tolerance for other packets sent by the stack + [ $delta -ge $expected ] && [ $delta -le $((expected + 100)) ] +} + +rmon_histogram() +{ + local iface=$1; shift + local neigh=$1; shift + local set=$1; shift + local nbuckets=0 + local step= + + RET=0 + + while read -r -a bucket; do + step="$set-pkts${bucket[0]}to${bucket[1]} on $iface" + + for if in $iface $neigh; do + if ! ensure_mtu $if ${bucket[0]}; then + log_test_skip "$if does not support the required MTU for $step" + return + fi + done + + if ! bucket_test $iface $neigh $set $nbuckets ${bucket[0]}; then + check_err 1 "$step failed" + return 1 + fi + log_test "$step" + nbuckets=$((nbuckets + 1)) + done < <(ethtool --json -S $iface --groups rmon | \ + jq -r ".[0].rmon[\"${set}-pktsNtoM\"][]|[.low, .high]|@tsv" 2>/dev/null) + + if [ $nbuckets -eq 0 ]; then + log_test_skip "$iface does not support $set histogram counters" + return + fi +} + +rmon_rx_histogram() +{ + rmon_histogram $h1 $h2 rx + rmon_histogram $h2 $h1 rx +} + +rmon_tx_histogram() +{ + rmon_histogram $h1 $h2 tx + rmon_histogram $h2 $h1 tx +} + +setup_prepare() +{ + h1=${NETIFS[p1]} + h2=${NETIFS[p2]} + + for iface in $h1 $h2; do + netif_mtu[$iface]=$(ip -j link show dev $iface | jq -r '.[0].mtu') + ip link set dev $iface up + done +} + +cleanup() +{ + pre_cleanup + + for iface in $h2 $h1; do + ip link set dev $iface \ + mtu ${netif_mtu[$iface]} \ + down + done +} + +check_ethtool_counter_group_support +trap cleanup EXIT + +setup_prepare +setup_wait + +tests_run + +exit $EXIT_STATUS diff --git a/tools/testing/selftests/net/forwarding/lib.sh b/tools/testing/selftests/net/forwarding/lib.sh index 8f6ca458af9a8d..e3740163c384fd 100755 --- a/tools/testing/selftests/net/forwarding/lib.sh +++ b/tools/testing/selftests/net/forwarding/lib.sh @@ -146,6 +146,15 @@ check_ethtool_mm_support() fi } +check_ethtool_counter_group_support() +{ + ethtool --help 2>&1| grep -- '--all-groups' &> /dev/null + if [[ $? -ne 0 ]]; then + echo "SKIP: ethtool too old; it is missing standard counter group support" + exit $ksft_skip + fi +} + check_locked_port_support() { if ! bridge -d link show | grep -q " locked"; then diff --git a/tools/testing/selftests/net/io_uring_zerocopy_tx.sh b/tools/testing/selftests/net/io_uring_zerocopy_tx.sh index 9ac4456d48fcc6..123439545013d0 100755 --- a/tools/testing/selftests/net/io_uring_zerocopy_tx.sh +++ b/tools/testing/selftests/net/io_uring_zerocopy_tx.sh @@ -76,23 +76,22 @@ case "${TXMODE}" in esac # Start of state changes: install cleanup handler -save_sysctl_mem="$(sysctl -n ${path_sysctl_mem})" cleanup() { ip netns del "${NS2}" ip netns del "${NS1}" - sysctl -w -q "${path_sysctl_mem}=${save_sysctl_mem}" } trap cleanup EXIT -# Configure system settings -sysctl -w -q "${path_sysctl_mem}=1000000" - # Create virtual ethernet pair between network namespaces ip netns add "${NS1}" ip netns add "${NS2}" +# Configure system settings +ip netns exec "${NS1}" sysctl -w -q "${path_sysctl_mem}=1000000" +ip netns exec "${NS2}" sysctl -w -q "${path_sysctl_mem}=1000000" + ip link add "${DEV}" mtu "${DEV_MTU}" netns "${NS1}" type veth \ peer name "${DEV}" mtu "${DEV_MTU}" netns "${NS2}" diff --git a/tools/testing/selftests/net/msg_zerocopy.sh b/tools/testing/selftests/net/msg_zerocopy.sh index 825ffec85cea3e..89c22f5320e0d6 100755 --- a/tools/testing/selftests/net/msg_zerocopy.sh +++ b/tools/testing/selftests/net/msg_zerocopy.sh @@ -70,23 +70,22 @@ case "${TXMODE}" in esac # Start of state changes: install cleanup handler -save_sysctl_mem="$(sysctl -n ${path_sysctl_mem})" cleanup() { ip netns del "${NS2}" ip netns del "${NS1}" - sysctl -w -q "${path_sysctl_mem}=${save_sysctl_mem}" } trap cleanup EXIT -# Configure system settings -sysctl -w -q "${path_sysctl_mem}=1000000" - # Create virtual ethernet pair between network namespaces ip netns add "${NS1}" ip netns add "${NS2}" +# Configure system settings +ip netns exec "${NS1}" sysctl -w -q "${path_sysctl_mem}=1000000" +ip netns exec "${NS2}" sysctl -w -q "${path_sysctl_mem}=1000000" + ip link add "${DEV}" mtu "${DEV_MTU}" netns "${NS1}" type veth \ peer name "${DEV}" mtu "${DEV_MTU}" netns "${NS2}" diff --git a/tools/testing/selftests/net/tcp_ao/.gitignore b/tools/testing/selftests/net/tcp_ao/.gitignore new file mode 100644 index 00000000000000..e8bb81b715b79b --- /dev/null +++ b/tools/testing/selftests/net/tcp_ao/.gitignore @@ -0,0 +1,2 @@ +*_ipv4 +*_ipv6 diff --git a/tools/testing/selftests/net/tcp_ao/Makefile b/tools/testing/selftests/net/tcp_ao/Makefile new file mode 100644 index 00000000000000..6343cfcf919b30 --- /dev/null +++ b/tools/testing/selftests/net/tcp_ao/Makefile @@ -0,0 +1,59 @@ +# SPDX-License-Identifier: GPL-2.0 +TEST_BOTH_AF := bench-lookups +TEST_BOTH_AF += connect +TEST_BOTH_AF += connect-deny +TEST_BOTH_AF += icmps-accept icmps-discard +TEST_BOTH_AF += key-management +TEST_BOTH_AF += restore +TEST_BOTH_AF += rst +TEST_BOTH_AF += self-connect +TEST_BOTH_AF += seq-ext +TEST_BOTH_AF += setsockopt-closed +TEST_BOTH_AF += unsigned-md5 + +TEST_IPV4_PROGS := $(TEST_BOTH_AF:%=%_ipv4) +TEST_IPV6_PROGS := $(TEST_BOTH_AF:%=%_ipv6) + +TEST_GEN_PROGS := $(TEST_IPV4_PROGS) $(TEST_IPV6_PROGS) + +top_srcdir := ../../../../.. +KSFT_KHDR_INSTALL := 1 +include ../../lib.mk + +HOSTAR ?= ar + +# Drop it on port to linux/master with commit 8ce72dc32578 +.DEFAULT_GOAL := all + +LIBDIR := $(OUTPUT)/lib +LIB := $(LIBDIR)/libaotst.a +LDLIBS += $(LIB) -pthread +LIBDEPS := lib/aolib.h Makefile + +CFLAGS := -Wall -O2 -g -D_GNU_SOURCE -fno-strict-aliasing +CFLAGS += -I ../../../../../usr/include/ -iquote $(LIBDIR) +CFLAGS += -I ../../../../include/ + +# Library +LIBSRC := kconfig.c netlink.c proc.c repair.c setup.c sock.c utils.c +LIBOBJ := $(LIBSRC:%.c=$(LIBDIR)/%.o) +EXTRA_CLEAN += $(LIBOBJ) $(LIB) + +$(LIB): $(LIBOBJ) + $(HOSTAR) rcs $@ $^ + +$(LIBDIR)/%.o: ./lib/%.c $(LIBDEPS) + $(CC) $< $(CFLAGS) $(CPPFLAGS) -o $@ -c + +$(TEST_GEN_PROGS): $(LIB) + +$(OUTPUT)/%_ipv4: %.c + $(LINK.c) $^ $(LDLIBS) -o $@ + +$(OUTPUT)/%_ipv6: %.c + $(LINK.c) -DIPV6_TEST $^ $(LDLIBS) -o $@ + +$(OUTPUT)/icmps-accept_ipv4: CFLAGS+= -DTEST_ICMPS_ACCEPT +$(OUTPUT)/icmps-accept_ipv6: CFLAGS+= -DTEST_ICMPS_ACCEPT +$(OUTPUT)/bench-lookups_ipv4: LDFLAGS+= -lm +$(OUTPUT)/bench-lookups_ipv6: LDFLAGS+= -lm diff --git a/tools/testing/selftests/net/tcp_ao/bench-lookups.c b/tools/testing/selftests/net/tcp_ao/bench-lookups.c new file mode 100644 index 00000000000000..7be8a7d9308c2d --- /dev/null +++ b/tools/testing/selftests/net/tcp_ao/bench-lookups.c @@ -0,0 +1,358 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Author: Dmitry Safonov */ +#include +#include +#include +#include +#include +#include + +#include "../../../../include/linux/bits.h" +#include "../../../../include/linux/kernel.h" +#include "aolib.h" + +#define BENCH_NR_ITERS 100 /* number of times to run gathering statistics */ + +static void gen_test_ips(union tcp_addr *ips, size_t ips_nr, bool use_rand) +{ + union tcp_addr net = {}; + size_t i, j; + + if (inet_pton(TEST_FAMILY, TEST_NETWORK, &net) != 1) + test_error("Can't convert ip address %s", TEST_NETWORK); + + if (!use_rand) { + for (i = 0; i < ips_nr; i++) + ips[i] = gen_tcp_addr(net, 2 * i + 1); + return; + } + for (i = 0; i < ips_nr; i++) { + size_t r = (size_t)random() | 0x1; + + ips[i] = gen_tcp_addr(net, r); + + for (j = i - 1; j > 0 && i > 0; j--) { + if (!memcmp(&ips[i], &ips[j], sizeof(union tcp_addr))) { + i--; /* collision */ + break; + } + } + } +} + +static void test_add_routes(union tcp_addr *ips, size_t ips_nr) +{ + size_t i; + + for (i = 0; i < ips_nr; i++) { + union tcp_addr *p = (union tcp_addr *)&ips[i]; + + if (ip_route_add(veth_name, TEST_FAMILY, this_ip_addr, *p)) + test_error("Failed to add route"); + } +} + +static void server_apply_keys(int lsk, union tcp_addr *ips, size_t ips_nr) +{ + size_t i; + + for (i = 0; i < ips_nr; i++) { + union tcp_addr *p = (union tcp_addr *)&ips[i]; + + if (test_add_key(lsk, DEFAULT_TEST_PASSWORD, *p, -1, 100, 100)) + test_error("setsockopt(TCP_AO)"); + } +} + +static const size_t nr_keys[] = { 512, 1024, 2048, 4096, 8192 }; +static union tcp_addr *test_ips; + +struct bench_stats { + uint64_t min; + uint64_t max; + uint64_t nr; + double mean; + double s2; +}; + +static struct bench_tests { + struct bench_stats delete_last_key; + struct bench_stats add_key; + struct bench_stats delete_rand_key; + struct bench_stats connect_last_key; + struct bench_stats connect_rand_key; + struct bench_stats delete_async; +} bench_results[ARRAY_SIZE(nr_keys)]; + +#define NSEC_PER_SEC 1000000000ULL + +static void measure_call(struct bench_stats *st, + void (*f)(int, void *), int sk, void *arg) +{ + struct timespec start = {}, end = {}; + double delta; + uint64_t nsec; + + if (clock_gettime(CLOCK_MONOTONIC, &start)) + test_error("clock_gettime()"); + + f(sk, arg); + + if (clock_gettime(CLOCK_MONOTONIC, &end)) + test_error("clock_gettime()"); + + nsec = (end.tv_sec - start.tv_sec) * NSEC_PER_SEC; + if (end.tv_nsec >= start.tv_nsec) + nsec += end.tv_nsec - start.tv_nsec; + else + nsec -= start.tv_nsec - end.tv_nsec; + + if (st->nr == 0) { + st->min = st->max = nsec; + } else { + if (st->min > nsec) + st->min = nsec; + if (st->max < nsec) + st->max = nsec; + } + + /* Welford-Knuth algorithm */ + st->nr++; + delta = (double)nsec - st->mean; + st->mean += delta / st->nr; + st->s2 += delta * ((double)nsec - st->mean); +} + +static void delete_mkt(int sk, void *arg) +{ + struct tcp_ao_del *ao = arg; + + if (setsockopt(sk, IPPROTO_TCP, TCP_AO_DEL_KEY, ao, sizeof(*ao))) + test_error("setsockopt(TCP_AO_DEL_KEY)"); +} + +static void add_back_mkt(int sk, void *arg) +{ + union tcp_addr *p = arg; + + if (test_add_key(sk, DEFAULT_TEST_PASSWORD, *p, -1, 100, 100)) + test_error("setsockopt(TCP_AO)"); +} + +static void bench_delete(int lsk, struct bench_stats *add, + struct bench_stats *del, + union tcp_addr *ips, size_t ips_nr, + bool rand_order, bool async) +{ + struct tcp_ao_del ao_del = {}; + union tcp_addr *p; + size_t i; + + ao_del.sndid = 100; + ao_del.rcvid = 100; + ao_del.del_async = !!async; + ao_del.prefix = DEFAULT_TEST_PREFIX; + + /* Remove the first added */ + p = (union tcp_addr *)&ips[0]; + tcp_addr_to_sockaddr_in(&ao_del.addr, p, 0); + + for (i = 0; i < BENCH_NR_ITERS; i++) { + measure_call(del, delete_mkt, lsk, (void *)&ao_del); + + /* Restore it back */ + measure_call(add, add_back_mkt, lsk, (void *)p); + + /* + * Slowest for FILO-linked-list: + * on (i) iteration removing ips[i] element. When it gets + * added to the list back - it becomes first to fetch, so + * on (i + 1) iteration go to ips[i + 1] element. + */ + if (rand_order) + p = (union tcp_addr *)&ips[rand() % ips_nr]; + else + p = (union tcp_addr *)&ips[i % ips_nr]; + tcp_addr_to_sockaddr_in(&ao_del.addr, p, 0); + } +} + +static void bench_connect_srv(int lsk, union tcp_addr *ips, size_t ips_nr) +{ + size_t i; + + for (i = 0; i < BENCH_NR_ITERS; i++) { + int sk; + + synchronize_threads(); + + if (test_wait_fd(lsk, TEST_TIMEOUT_SEC, 0)) + test_error("test_wait_fd()"); + + sk = accept(lsk, NULL, NULL); + if (sk < 0) + test_error("accept()"); + + close(sk); + } +} + +static void test_print_stats(const char *desc, size_t nr, struct bench_stats *bs) +{ + test_ok("%-20s\t%zu keys: min=%" PRIu64 "ms max=%" PRIu64 "ms mean=%gms stddev=%g", + desc, nr, bs->min / 1000000, bs->max / 1000000, + bs->mean / 1000000, sqrt((bs->mean / 1000000) / bs->nr)); +} + +static void *server_fn(void *arg) +{ + size_t i; + + for (i = 0; i < ARRAY_SIZE(nr_keys); i++) { + struct bench_tests *bt = &bench_results[i]; + int lsk; + + test_ips = malloc(nr_keys[i] * sizeof(union tcp_addr)); + if (!test_ips) + test_error("malloc()"); + + lsk = test_listen_socket(this_ip_addr, test_server_port + i, 1); + + gen_test_ips(test_ips, nr_keys[i], false); + test_add_routes(test_ips, nr_keys[i]); + test_set_optmem(KERNEL_TCP_AO_KEY_SZ_ROUND_UP * nr_keys[i]); + server_apply_keys(lsk, test_ips, nr_keys[i]); + + synchronize_threads(); + bench_connect_srv(lsk, test_ips, nr_keys[i]); + bench_connect_srv(lsk, test_ips, nr_keys[i]); + + /* The worst case for FILO-list */ + bench_delete(lsk, &bt->add_key, &bt->delete_last_key, + test_ips, nr_keys[i], false, false); + test_print_stats("Add a new key", + nr_keys[i], &bt->add_key); + test_print_stats("Delete: worst case", + nr_keys[i], &bt->delete_last_key); + + bench_delete(lsk, &bt->add_key, &bt->delete_rand_key, + test_ips, nr_keys[i], true, false); + test_print_stats("Delete: random-search", + nr_keys[i], &bt->delete_rand_key); + + bench_delete(lsk, &bt->add_key, &bt->delete_async, + test_ips, nr_keys[i], false, true); + test_print_stats("Delete: async", nr_keys[i], &bt->delete_async); + + free(test_ips); + close(lsk); + } + + return NULL; +} + +static void connect_client(int sk, void *arg) +{ + size_t *p = arg; + + if (test_connect_socket(sk, this_ip_dest, test_server_port + *p) <= 0) + test_error("failed to connect()"); +} + +static void client_addr_setup(int sk, union tcp_addr taddr) +{ +#ifdef IPV6_TEST + struct sockaddr_in6 addr = { + .sin6_family = AF_INET6, + .sin6_port = 0, + .sin6_addr = taddr.a6, + }; +#else + struct sockaddr_in addr = { + .sin_family = AF_INET, + .sin_port = 0, + .sin_addr = taddr.a4, + }; +#endif + int ret; + + ret = ip_addr_add(veth_name, TEST_FAMILY, taddr, TEST_PREFIX); + if (ret && ret != -EEXIST) + test_error("Failed to add ip address"); + ret = ip_route_add(veth_name, TEST_FAMILY, taddr, this_ip_dest); + if (ret && ret != -EEXIST) + test_error("Failed to add route"); + + if (bind(sk, &addr, sizeof(addr))) + test_error("bind()"); +} + +static void bench_connect_client(size_t port_off, struct bench_tests *bt, + union tcp_addr *ips, size_t ips_nr, bool rand_order) +{ + struct bench_stats *con; + union tcp_addr *p; + size_t i; + + if (rand_order) + con = &bt->connect_rand_key; + else + con = &bt->connect_last_key; + + p = (union tcp_addr *)&ips[0]; + + for (i = 0; i < BENCH_NR_ITERS; i++) { + int sk = socket(test_family, SOCK_STREAM, IPPROTO_TCP); + + if (sk < 0) + test_error("socket()"); + + client_addr_setup(sk, *p); + if (test_add_key(sk, DEFAULT_TEST_PASSWORD, this_ip_dest, + -1, 100, 100)) + test_error("setsockopt(TCP_AO_ADD_KEY)"); + + synchronize_threads(); + + measure_call(con, connect_client, sk, (void *)&port_off); + + close(sk); + + /* + * Slowest for FILO-linked-list: + * on (i) iteration removing ips[i] element. When it gets + * added to the list back - it becomes first to fetch, so + * on (i + 1) iteration go to ips[i + 1] element. + */ + if (rand_order) + p = (union tcp_addr *)&ips[rand() % ips_nr]; + else + p = (union tcp_addr *)&ips[i % ips_nr]; + } +} + +static void *client_fn(void *arg) +{ + size_t i; + + for (i = 0; i < ARRAY_SIZE(nr_keys); i++) { + struct bench_tests *bt = &bench_results[i]; + + synchronize_threads(); + bench_connect_client(i, bt, test_ips, nr_keys[i], false); + test_print_stats("Connect: worst case", + nr_keys[i], &bt->connect_last_key); + + bench_connect_client(i, bt, test_ips, nr_keys[i], false); + test_print_stats("Connect: random-search", + nr_keys[i], &bt->connect_last_key); + } + synchronize_threads(); + return NULL; +} + +int main(int argc, char *argv[]) +{ + test_init(30, server_fn, client_fn); + return 0; +} diff --git a/tools/testing/selftests/net/tcp_ao/connect-deny.c b/tools/testing/selftests/net/tcp_ao/connect-deny.c new file mode 100644 index 00000000000000..1ca78040d8b7ed --- /dev/null +++ b/tools/testing/selftests/net/tcp_ao/connect-deny.c @@ -0,0 +1,264 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Author: Dmitry Safonov */ +#include +#include "aolib.h" + +#define fault(type) (inj == FAULT_ ## type) + +static inline int test_add_key_maclen(int sk, const char *key, uint8_t maclen, + union tcp_addr in_addr, uint8_t prefix, + uint8_t sndid, uint8_t rcvid) +{ + struct tcp_ao_add tmp = {}; + int err; + + if (prefix > DEFAULT_TEST_PREFIX) + prefix = DEFAULT_TEST_PREFIX; + + err = test_prepare_key(&tmp, DEFAULT_TEST_ALGO, in_addr, false, false, + prefix, 0, sndid, rcvid, maclen, + 0, strlen(key), key); + if (err) + return err; + + err = setsockopt(sk, IPPROTO_TCP, TCP_AO_ADD_KEY, &tmp, sizeof(tmp)); + if (err < 0) + return -errno; + + return test_verify_socket_key(sk, &tmp); +} + +static void try_accept(const char *tst_name, unsigned int port, const char *pwd, + union tcp_addr addr, uint8_t prefix, + uint8_t sndid, uint8_t rcvid, uint8_t maclen, + const char *cnt_name, test_cnt cnt_expected, + fault_t inj) +{ + struct tcp_ao_counters ao_cnt1, ao_cnt2; + uint64_t before_cnt = 0, after_cnt = 0; /* silence GCC */ + int lsk, err, sk = 0; + time_t timeout; + + lsk = test_listen_socket(this_ip_addr, port, 1); + + if (pwd && test_add_key_maclen(lsk, pwd, maclen, addr, prefix, sndid, rcvid)) + test_error("setsockopt(TCP_AO_ADD_KEY)"); + + if (cnt_name) + before_cnt = netstat_get_one(cnt_name, NULL); + if (pwd && test_get_tcp_ao_counters(lsk, &ao_cnt1)) + test_error("test_get_tcp_ao_counters()"); + + synchronize_threads(); /* preparations done */ + + timeout = fault(TIMEOUT) ? TEST_RETRANSMIT_SEC : TEST_TIMEOUT_SEC; + err = test_wait_fd(lsk, timeout, 0); + if (err == -ETIMEDOUT) { + if (!fault(TIMEOUT)) + test_fail("timeouted for accept()"); + } else if (err < 0) { + test_error("test_wait_fd()"); + } else { + if (fault(TIMEOUT)) + test_fail("ready to accept"); + + sk = accept(lsk, NULL, NULL); + if (sk < 0) { + test_error("accept()"); + } else { + if (fault(TIMEOUT)) + test_fail("%s: accepted", tst_name); + } + } + + if (pwd && test_get_tcp_ao_counters(lsk, &ao_cnt2)) + test_error("test_get_tcp_ao_counters()"); + + close(lsk); + if (pwd) + test_tcp_ao_counters_cmp(tst_name, &ao_cnt1, &ao_cnt2, cnt_expected); + + if (!cnt_name) + goto out; + + after_cnt = netstat_get_one(cnt_name, NULL); + + if (after_cnt <= before_cnt) { + test_fail("%s: %s counter did not increase: %zu <= %zu", + tst_name, cnt_name, after_cnt, before_cnt); + } else { + test_ok("%s: counter %s increased %zu => %zu", + tst_name, cnt_name, before_cnt, after_cnt); + } + +out: + synchronize_threads(); /* close() */ + if (sk > 0) + close(sk); +} + +static void *server_fn(void *arg) +{ + union tcp_addr wrong_addr, network_addr; + unsigned int port = test_server_port; + + if (inet_pton(TEST_FAMILY, TEST_WRONG_IP, &wrong_addr) != 1) + test_error("Can't convert ip address %s", TEST_WRONG_IP); + + try_accept("Non-AO server + AO client", port++, NULL, + this_ip_dest, -1, 100, 100, 0, + "TCPAOKeyNotFound", 0, FAULT_TIMEOUT); + + try_accept("AO server + Non-AO client", port++, DEFAULT_TEST_PASSWORD, + this_ip_dest, -1, 100, 100, 0, + "TCPAORequired", TEST_CNT_AO_REQUIRED, FAULT_TIMEOUT); + + try_accept("Wrong password", port++, "something that is not DEFAULT_TEST_PASSWORD", + this_ip_dest, -1, 100, 100, 0, + "TCPAOBad", TEST_CNT_BAD, FAULT_TIMEOUT); + + try_accept("Wrong rcv id", port++, DEFAULT_TEST_PASSWORD, + this_ip_dest, -1, 100, 101, 0, + "TCPAOKeyNotFound", TEST_CNT_AO_KEY_NOT_FOUND, FAULT_TIMEOUT); + + try_accept("Wrong snd id", port++, DEFAULT_TEST_PASSWORD, + this_ip_dest, -1, 101, 100, 0, + "TCPAOGood", TEST_CNT_GOOD, FAULT_TIMEOUT); + + try_accept("Different maclen", port++, DEFAULT_TEST_PASSWORD, + this_ip_dest, -1, 100, 100, 8, + "TCPAOBad", TEST_CNT_BAD, FAULT_TIMEOUT); + + try_accept("Server: Wrong addr", port++, DEFAULT_TEST_PASSWORD, + wrong_addr, -1, 100, 100, 0, + "TCPAOKeyNotFound", TEST_CNT_AO_KEY_NOT_FOUND, FAULT_TIMEOUT); + + try_accept("Client: Wrong addr", port++, NULL, + this_ip_dest, -1, 100, 100, 0, NULL, 0, FAULT_TIMEOUT); + + try_accept("rcv id != snd id", port++, DEFAULT_TEST_PASSWORD, + this_ip_dest, -1, 200, 100, 0, + "TCPAOGood", TEST_CNT_GOOD, 0); + + if (inet_pton(TEST_FAMILY, TEST_NETWORK, &network_addr) != 1) + test_error("Can't convert ip address %s", TEST_NETWORK); + + try_accept("Server: prefix match", port++, DEFAULT_TEST_PASSWORD, + network_addr, 16, 100, 100, 0, + "TCPAOGood", TEST_CNT_GOOD, 0); + + try_accept("Client: prefix match", port++, DEFAULT_TEST_PASSWORD, + this_ip_dest, -1, 100, 100, 0, + "TCPAOGood", TEST_CNT_GOOD, 0); + + /* client exits */ + synchronize_threads(); + return NULL; +} + +static void try_connect(const char *tst_name, unsigned int port, + const char *pwd, union tcp_addr addr, uint8_t prefix, + uint8_t sndid, uint8_t rcvid, + test_cnt cnt_expected, fault_t inj) +{ + struct tcp_ao_counters ao_cnt1, ao_cnt2; + time_t timeout; + int sk, ret; + + sk = socket(test_family, SOCK_STREAM, IPPROTO_TCP); + if (sk < 0) + test_error("socket()"); + + if (pwd && test_add_key(sk, pwd, addr, prefix, sndid, rcvid)) + test_error("setsockopt(TCP_AO_ADD_KEY)"); + + if (pwd && test_get_tcp_ao_counters(sk, &ao_cnt1)) + test_error("test_get_tcp_ao_counters()"); + + synchronize_threads(); /* preparations done */ + + timeout = fault(TIMEOUT) ? TEST_RETRANSMIT_SEC : TEST_TIMEOUT_SEC; + ret = _test_connect_socket(sk, this_ip_dest, port, timeout); + + if (ret < 0) { + if (fault(KEYREJECT) && ret == -EKEYREJECTED) { + test_ok("%s: connect() was prevented", tst_name); + } else if (ret == -ETIMEDOUT && fault(TIMEOUT)) { + test_ok("%s", tst_name); + } else if (ret == -ECONNREFUSED && + (fault(TIMEOUT) || fault(KEYREJECT))) { + test_ok("%s: refused to connect", tst_name); + } else { + test_error("%s: connect() returned %d", tst_name, ret); + } + goto out; + } + + if (fault(TIMEOUT) || fault(KEYREJECT)) + test_fail("%s: connected", tst_name); + else + test_ok("%s: connected", tst_name); + if (pwd && ret > 0) { + if (test_get_tcp_ao_counters(sk, &ao_cnt2)) + test_error("test_get_tcp_ao_counters()"); + test_tcp_ao_counters_cmp(tst_name, &ao_cnt1, &ao_cnt2, cnt_expected); + } +out: + synchronize_threads(); /* close() */ + + if (ret > 0) + close(sk); +} + +static void *client_fn(void *arg) +{ + union tcp_addr wrong_addr, network_addr; + unsigned int port = test_server_port; + + if (inet_pton(TEST_FAMILY, TEST_WRONG_IP, &wrong_addr) != 1) + test_error("Can't convert ip address %s", TEST_WRONG_IP); + + try_connect("Non-AO server + AO client", port++, DEFAULT_TEST_PASSWORD, + this_ip_dest, -1, 100, 100, 0, FAULT_TIMEOUT); + + try_connect("AO server + Non-AO client", port++, NULL, + this_ip_dest, -1, 100, 100, 0, FAULT_TIMEOUT); + + try_connect("Wrong password", port++, DEFAULT_TEST_PASSWORD, + this_ip_dest, -1, 100, 100, 0, FAULT_TIMEOUT); + + try_connect("Wrong rcv id", port++, DEFAULT_TEST_PASSWORD, + this_ip_dest, -1, 100, 100, 0, FAULT_TIMEOUT); + + try_connect("Wrong snd id", port++, DEFAULT_TEST_PASSWORD, + this_ip_dest, -1, 100, 100, 0, FAULT_TIMEOUT); + + try_connect("Different maclen", port++, DEFAULT_TEST_PASSWORD, + this_ip_dest, -1, 100, 100, 0, FAULT_TIMEOUT); + + try_connect("Server: Wrong addr", port++, DEFAULT_TEST_PASSWORD, + this_ip_dest, -1, 100, 100, 0, FAULT_TIMEOUT); + + try_connect("Client: Wrong addr", port++, DEFAULT_TEST_PASSWORD, + wrong_addr, -1, 100, 100, 0, FAULT_KEYREJECT); + + try_connect("rcv id != snd id", port++, DEFAULT_TEST_PASSWORD, + this_ip_dest, -1, 100, 200, TEST_CNT_GOOD, 0); + + if (inet_pton(TEST_FAMILY, TEST_NETWORK, &network_addr) != 1) + test_error("Can't convert ip address %s", TEST_NETWORK); + + try_connect("Server: prefix match", port++, DEFAULT_TEST_PASSWORD, + this_ip_dest, -1, 100, 100, TEST_CNT_GOOD, 0); + + try_connect("Client: prefix match", port++, DEFAULT_TEST_PASSWORD, + network_addr, 16, 100, 100, TEST_CNT_GOOD, 0); + + return NULL; +} + +int main(int argc, char *argv[]) +{ + test_init(21, server_fn, client_fn); + return 0; +} diff --git a/tools/testing/selftests/net/tcp_ao/connect.c b/tools/testing/selftests/net/tcp_ao/connect.c new file mode 100644 index 00000000000000..81653b47f303c1 --- /dev/null +++ b/tools/testing/selftests/net/tcp_ao/connect.c @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Author: Dmitry Safonov */ +#include +#include "aolib.h" + +static void *server_fn(void *arg) +{ + int sk, lsk; + ssize_t bytes; + + lsk = test_listen_socket(this_ip_addr, test_server_port, 1); + + if (test_add_key(lsk, DEFAULT_TEST_PASSWORD, this_ip_dest, -1, 100, 100)) + test_error("setsockopt(TCP_AO_ADD_KEY)"); + synchronize_threads(); + + if (test_wait_fd(lsk, TEST_TIMEOUT_SEC, 0)) + test_error("test_wait_fd()"); + + sk = accept(lsk, NULL, NULL); + if (sk < 0) + test_error("accept()"); + + synchronize_threads(); + + bytes = test_server_run(sk, 0, 0); + + test_fail("server served: %zd", bytes); + return NULL; +} + +static void *client_fn(void *arg) +{ + int sk = socket(test_family, SOCK_STREAM, IPPROTO_TCP); + uint64_t before_aogood, after_aogood; + const size_t nr_packets = 20; + struct netstat *ns_before, *ns_after; + struct tcp_ao_counters ao1, ao2; + + if (sk < 0) + test_error("socket()"); + + if (test_add_key(sk, DEFAULT_TEST_PASSWORD, this_ip_dest, -1, 100, 100)) + test_error("setsockopt(TCP_AO_ADD_KEY)"); + + synchronize_threads(); + if (test_connect_socket(sk, this_ip_dest, test_server_port) <= 0) + test_error("failed to connect()"); + synchronize_threads(); + + ns_before = netstat_read(); + before_aogood = netstat_get(ns_before, "TCPAOGood", NULL); + if (test_get_tcp_ao_counters(sk, &ao1)) + test_error("test_get_tcp_ao_counters()"); + + if (test_client_verify(sk, 100, nr_packets, TEST_TIMEOUT_SEC)) { + test_fail("verify failed"); + return NULL; + } + + ns_after = netstat_read(); + after_aogood = netstat_get(ns_after, "TCPAOGood", NULL); + if (test_get_tcp_ao_counters(sk, &ao2)) + test_error("test_get_tcp_ao_counters()"); + netstat_print_diff(ns_before, ns_after); + netstat_free(ns_before); + netstat_free(ns_after); + + if (nr_packets > (after_aogood - before_aogood)) { + test_fail("TCPAOGood counter mismatch: %zu > (%zu - %zu)", + nr_packets, after_aogood, before_aogood); + return NULL; + } + if (test_tcp_ao_counters_cmp("connect", &ao1, &ao2, TEST_CNT_GOOD)) + return NULL; + + test_ok("connect TCPAOGood %" PRIu64 "/%" PRIu64 "/%" PRIu64 " => %" PRIu64 "/%" PRIu64 "/%" PRIu64 ", sent %" PRIu64, + before_aogood, ao1.ao_info_pkt_good, + ao1.key_cnts[0].pkt_good, + after_aogood, ao2.ao_info_pkt_good, + ao2.key_cnts[0].pkt_good, + nr_packets); + return NULL; +} + +int main(int argc, char *argv[]) +{ + test_init(1, server_fn, client_fn); + return 0; +} diff --git a/tools/testing/selftests/net/tcp_ao/icmps-accept.c b/tools/testing/selftests/net/tcp_ao/icmps-accept.c new file mode 120000 index 00000000000000..0a5bb85eb26073 --- /dev/null +++ b/tools/testing/selftests/net/tcp_ao/icmps-accept.c @@ -0,0 +1 @@ +icmps-discard.c \ No newline at end of file diff --git a/tools/testing/selftests/net/tcp_ao/icmps-discard.c b/tools/testing/selftests/net/tcp_ao/icmps-discard.c new file mode 100644 index 00000000000000..d69bcba3c929d6 --- /dev/null +++ b/tools/testing/selftests/net/tcp_ao/icmps-discard.c @@ -0,0 +1,449 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Selftest that verifies that incomping ICMPs are ignored, + * the TCP connection stays alive, no hard or soft errors get reported + * to the usespace and the counter for ignored ICMPs is updated. + * + * RFC5925, 7.8: + * >> A TCP-AO implementation MUST default to ignore incoming ICMPv4 + * messages of Type 3 (destination unreachable), Codes 2-4 (protocol + * unreachable, port unreachable, and fragmentation needed -- ’hard + * errors’), and ICMPv6 Type 1 (destination unreachable), Code 1 + * (administratively prohibited) and Code 4 (port unreachable) intended + * for connections in synchronized states (ESTABLISHED, FIN-WAIT-1, FIN- + * WAIT-2, CLOSE-WAIT, CLOSING, LAST-ACK, TIME-WAIT) that match MKTs. + * + * Author: Dmitry Safonov + */ +#include +#include +#include +#include +#include +#include +#include +#include "aolib.h" +#include "../../../../include/linux/compiler.h" + +const size_t packets_nr = 20; +const size_t packet_size = 100; +const char *tcpao_icmps = "TCPAODroppedIcmps"; + +#ifdef IPV6_TEST +const char *dst_unreach = "Icmp6InDestUnreachs"; +const int sk_ip_level = SOL_IPV6; +const int sk_recverr = IPV6_RECVERR; +#else +const char *dst_unreach = "InDestUnreachs"; +const int sk_ip_level = SOL_IP; +const int sk_recverr = IP_RECVERR; +#endif + +/* Server is expected to fail with hard error if ::accept_icmp is set */ +#ifdef TEST_ICMPS_ACCEPT +# define test_icmps_fail test_ok +# define test_icmps_ok test_fail +#else +# define test_icmps_fail test_fail +# define test_icmps_ok test_ok +#endif + +static void serve_interfered(int sk) +{ + ssize_t test_quota = packet_size * packets_nr * 10; + uint64_t dest_unreach_a, dest_unreach_b; + uint64_t icmp_ignored_a, icmp_ignored_b; + struct tcp_ao_counters ao_cnt1, ao_cnt2; + bool counter_not_found; + struct netstat *ns_after, *ns_before; + ssize_t bytes; + + ns_before = netstat_read(); + dest_unreach_a = netstat_get(ns_before, dst_unreach, NULL); + icmp_ignored_a = netstat_get(ns_before, tcpao_icmps, NULL); + if (test_get_tcp_ao_counters(sk, &ao_cnt1)) + test_error("test_get_tcp_ao_counters()"); + bytes = test_server_run(sk, test_quota, 0); + ns_after = netstat_read(); + netstat_print_diff(ns_before, ns_after); + dest_unreach_b = netstat_get(ns_after, dst_unreach, NULL); + icmp_ignored_b = netstat_get(ns_after, tcpao_icmps, + &counter_not_found); + if (test_get_tcp_ao_counters(sk, &ao_cnt2)) + test_error("test_get_tcp_ao_counters()"); + + netstat_free(ns_before); + netstat_free(ns_after); + + if (dest_unreach_a >= dest_unreach_b) { + test_fail("%s counter didn't change: %" PRIu64 " >= %" PRIu64, + dst_unreach, dest_unreach_a, dest_unreach_b); + return; + } + test_ok("%s delivered %" PRIu64, + dst_unreach, dest_unreach_b - dest_unreach_a); + if (bytes < 0) + test_icmps_fail("Server failed with %zd: %s", bytes, strerrordesc_np(-bytes)); + else + test_icmps_ok("Server survived %zd bytes of traffic", test_quota); + if (counter_not_found) { + test_fail("Not found %s counter", tcpao_icmps); + return; + } +#ifdef TEST_ICMPS_ACCEPT + test_tcp_ao_counters_cmp(NULL, &ao_cnt1, &ao_cnt2, TEST_CNT_GOOD); +#else + test_tcp_ao_counters_cmp(NULL, &ao_cnt1, &ao_cnt2, TEST_CNT_GOOD | TEST_CNT_AO_DROPPED_ICMP); +#endif + if (icmp_ignored_a >= icmp_ignored_b) { + test_icmps_fail("%s counter didn't change: %" PRIu64 " >= %" PRIu64, + tcpao_icmps, icmp_ignored_a, icmp_ignored_b); + return; + } + test_icmps_ok("ICMPs ignored %" PRIu64, icmp_ignored_b - icmp_ignored_a); +} + +static void *server_fn(void *arg) +{ + int val, sk, lsk; + bool accept_icmps = false; + + lsk = test_listen_socket(this_ip_addr, test_server_port, 1); + +#ifdef TEST_ICMPS_ACCEPT + accept_icmps = true; +#endif + + if (test_set_ao_flags(lsk, false, accept_icmps)) + test_error("setsockopt(TCP_AO_INFO)"); + + if (test_add_key(lsk, DEFAULT_TEST_PASSWORD, this_ip_dest, -1, 100, 100)) + test_error("setsockopt(TCP_AO_ADD_KEY)"); + synchronize_threads(); + + if (test_wait_fd(lsk, TEST_TIMEOUT_SEC, 0)) + test_error("test_wait_fd()"); + + sk = accept(lsk, NULL, NULL); + if (sk < 0) + test_error("accept()"); + + /* Fail on hard ip errors, such as dest unreachable (RFC1122) */ + val = 1; + if (setsockopt(sk, sk_ip_level, sk_recverr, &val, sizeof(val))) + test_error("setsockopt()"); + + synchronize_threads(); + + serve_interfered(sk); + return NULL; +} + +static size_t packets_sent; +static size_t icmps_sent; + +static uint32_t checksum4_nofold(void *data, size_t len, uint32_t sum) +{ + uint16_t *words = data; + size_t i; + + for (i = 0; i < len / sizeof(uint16_t); i++) + sum += words[i]; + if (len & 1) + sum += ((char *)data)[len - 1]; + return sum; +} + +static uint16_t checksum4_fold(void *data, size_t len, uint32_t sum) +{ + sum = checksum4_nofold(data, len, sum); + while (sum > 0xFFFF) + sum = (sum & 0xFFFF) + (sum >> 16); + return ~sum; +} + +static void set_ip4hdr(struct iphdr *iph, size_t packet_len, int proto, + struct sockaddr_in *src, struct sockaddr_in *dst) +{ + iph->version = 4; + iph->ihl = 5; + iph->tos = 0; + iph->tot_len = htons(packet_len); + iph->ttl = 2; + iph->protocol = proto; + iph->saddr = src->sin_addr.s_addr; + iph->daddr = dst->sin_addr.s_addr; + iph->check = checksum4_fold((void *)iph, iph->ihl << 1, 0); +} + +static void icmp_interfere4(uint8_t type, uint8_t code, uint32_t rcv_nxt, + struct sockaddr_in *src, struct sockaddr_in *dst) +{ + int sk = socket(AF_INET, SOCK_RAW, IPPROTO_RAW); + struct { + struct iphdr iph; + struct icmphdr icmph; + struct iphdr iphe; + struct { + uint16_t sport; + uint16_t dport; + uint32_t seq; + } tcph; + } packet = {}; + size_t packet_len; + ssize_t bytes; + + if (sk < 0) + test_error("socket(AF_INET, SOCK_RAW, IPPROTO_RAW)"); + + packet_len = sizeof(packet); + set_ip4hdr(&packet.iph, packet_len, IPPROTO_ICMP, src, dst); + + packet.icmph.type = type; + packet.icmph.code = code; + if (code == ICMP_FRAG_NEEDED) { + randomize_buffer(&packet.icmph.un.frag.mtu, + sizeof(packet.icmph.un.frag.mtu)); + } + + packet_len = sizeof(packet.iphe) + sizeof(packet.tcph); + set_ip4hdr(&packet.iphe, packet_len, IPPROTO_TCP, dst, src); + + packet.tcph.sport = dst->sin_port; + packet.tcph.dport = src->sin_port; + packet.tcph.seq = htonl(rcv_nxt); + + packet_len = sizeof(packet) - sizeof(packet.iph); + packet.icmph.checksum = checksum4_fold((void *)&packet.icmph, + packet_len, 0); + + bytes = sendto(sk, &packet, sizeof(packet), 0, + (struct sockaddr *)dst, sizeof(*dst)); + if (bytes != sizeof(packet)) + test_error("send(): %zd", bytes); + icmps_sent++; + + close(sk); +} + +static void set_ip6hdr(struct ipv6hdr *iph, size_t packet_len, int proto, + struct sockaddr_in6 *src, struct sockaddr_in6 *dst) +{ + iph->version = 6; + iph->payload_len = htons(packet_len); + iph->nexthdr = proto; + iph->hop_limit = 2; + iph->saddr = src->sin6_addr; + iph->daddr = dst->sin6_addr; +} + +static inline uint16_t csum_fold(uint32_t csum) +{ + uint32_t sum = csum; + + sum = (sum & 0xffff) + (sum >> 16); + sum = (sum & 0xffff) + (sum >> 16); + return (uint16_t)~sum; +} + +static inline uint32_t csum_add(uint32_t csum, uint32_t addend) +{ + uint32_t res = csum; + + res += addend; + return res + (res < addend); +} + +noinline uint32_t checksum6_nofold(void *data, size_t len, uint32_t sum) +{ + uint16_t *words = data; + size_t i; + + for (i = 0; i < len / sizeof(uint16_t); i++) + sum = csum_add(sum, words[i]); + if (len & 1) + sum = csum_add(sum, ((char *)data)[len - 1]); + return sum; +} + +noinline uint16_t icmp6_checksum(struct sockaddr_in6 *src, + struct sockaddr_in6 *dst, + void *ptr, size_t len, uint8_t proto) +{ + struct { + struct in6_addr saddr; + struct in6_addr daddr; + uint32_t payload_len; + uint8_t zero[3]; + uint8_t nexthdr; + } pseudo_header = {}; + uint32_t sum; + + pseudo_header.saddr = src->sin6_addr; + pseudo_header.daddr = dst->sin6_addr; + pseudo_header.payload_len = htonl(len); + pseudo_header.nexthdr = proto; + + sum = checksum6_nofold(&pseudo_header, sizeof(pseudo_header), 0); + sum = checksum6_nofold(ptr, len, sum); + + return csum_fold(sum); +} + +static void icmp6_interfere(int type, int code, uint32_t rcv_nxt, + struct sockaddr_in6 *src, struct sockaddr_in6 *dst) +{ + int sk = socket(AF_INET6, SOCK_RAW, IPPROTO_RAW); + struct sockaddr_in6 dst_raw = *dst; + struct { + struct ipv6hdr iph; + struct icmp6hdr icmph; + struct ipv6hdr iphe; + struct { + uint16_t sport; + uint16_t dport; + uint32_t seq; + } tcph; + } packet = {}; + size_t packet_len; + ssize_t bytes; + + + if (sk < 0) + test_error("socket(AF_INET6, SOCK_RAW, IPPROTO_RAW)"); + + packet_len = sizeof(packet) - sizeof(packet.iph); + set_ip6hdr(&packet.iph, packet_len, IPPROTO_ICMPV6, src, dst); + + packet.icmph.icmp6_type = type; + packet.icmph.icmp6_code = code; + + packet_len = sizeof(packet.iphe) + sizeof(packet.tcph); + set_ip6hdr(&packet.iphe, packet_len, IPPROTO_TCP, dst, src); + + packet.tcph.sport = dst->sin6_port; + packet.tcph.dport = src->sin6_port; + packet.tcph.seq = htonl(rcv_nxt); + + packet_len = sizeof(packet) - sizeof(packet.iph); + + packet.icmph.icmp6_cksum = icmp6_checksum(src, dst, + (void *)&packet.icmph, packet_len, IPPROTO_ICMPV6); + + dst_raw.sin6_port = htons(IPPROTO_RAW); + bytes = sendto(sk, &packet, sizeof(packet), 0, + (struct sockaddr *)&dst_raw, sizeof(dst_raw)); + if (bytes != sizeof(packet)) + test_error("send(): %zd", bytes); + icmps_sent++; + + close(sk); +} + +static uint32_t get_rcv_nxt(int sk) +{ + int val = TCP_REPAIR_ON; + uint32_t ret; + socklen_t sz = sizeof(ret); + + if (setsockopt(sk, SOL_TCP, TCP_REPAIR, &val, sizeof(val))) + test_error("setsockopt(TCP_REPAIR)"); + val = TCP_RECV_QUEUE; + if (setsockopt(sk, SOL_TCP, TCP_REPAIR_QUEUE, &val, sizeof(val))) + test_error("setsockopt(TCP_REPAIR_QUEUE)"); + if (getsockopt(sk, SOL_TCP, TCP_QUEUE_SEQ, &ret, &sz)) + test_error("getsockopt(TCP_QUEUE_SEQ)"); + val = TCP_REPAIR_OFF_NO_WP; + if (setsockopt(sk, SOL_TCP, TCP_REPAIR, &val, sizeof(val))) + test_error("setsockopt(TCP_REPAIR)"); + return ret; +} + +static void icmp_interfere(const size_t nr, uint32_t rcv_nxt, void *src, void *dst) +{ + struct sockaddr_in *saddr4 = src; + struct sockaddr_in *daddr4 = dst; + struct sockaddr_in6 *saddr6 = src; + struct sockaddr_in6 *daddr6 = dst; + size_t i; + + if (saddr4->sin_family != daddr4->sin_family) + test_error("Different address families"); + + for (i = 0; i < nr; i++) { + if (saddr4->sin_family == AF_INET) { + icmp_interfere4(ICMP_DEST_UNREACH, ICMP_PROT_UNREACH, + rcv_nxt, saddr4, daddr4); + icmp_interfere4(ICMP_DEST_UNREACH, ICMP_PORT_UNREACH, + rcv_nxt, saddr4, daddr4); + icmp_interfere4(ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED, + rcv_nxt, saddr4, daddr4); + icmps_sent += 3; + } else if (saddr4->sin_family == AF_INET6) { + icmp6_interfere(ICMPV6_DEST_UNREACH, + ICMPV6_ADM_PROHIBITED, + rcv_nxt, saddr6, daddr6); + icmp6_interfere(ICMPV6_DEST_UNREACH, + ICMPV6_PORT_UNREACH, + rcv_nxt, saddr6, daddr6); + icmps_sent += 2; + } else { + test_error("Not ip address family"); + } + } +} + +static void send_interfered(int sk) +{ + const unsigned int timeout = TEST_TIMEOUT_SEC; + struct sockaddr_in6 src, dst; + socklen_t addr_sz; + + addr_sz = sizeof(src); + if (getsockname(sk, &src, &addr_sz)) + test_error("getsockname()"); + addr_sz = sizeof(dst); + if (getpeername(sk, &dst, &addr_sz)) + test_error("getpeername()"); + + while (1) { + uint32_t rcv_nxt; + + if (test_client_verify(sk, packet_size, packets_nr, timeout)) { + test_fail("client: connection is broken"); + return; + } + packets_sent += packets_nr; + rcv_nxt = get_rcv_nxt(sk); + icmp_interfere(packets_nr, rcv_nxt, (void *)&src, (void *)&dst); + } +} + +static void *client_fn(void *arg) +{ + int sk = socket(test_family, SOCK_STREAM, IPPROTO_TCP); + + if (sk < 0) + test_error("socket()"); + + if (test_add_key(sk, DEFAULT_TEST_PASSWORD, this_ip_dest, -1, 100, 100)) + test_error("setsockopt(TCP_AO_ADD_KEY)"); + + synchronize_threads(); + if (test_connect_socket(sk, this_ip_dest, test_server_port) <= 0) + test_error("failed to connect()"); + synchronize_threads(); + + send_interfered(sk); + + /* Not expecting client to quit */ + test_fail("client disconnected"); + + return NULL; +} + +int main(int argc, char *argv[]) +{ + test_init(3, server_fn, client_fn); + return 0; +} diff --git a/tools/testing/selftests/net/tcp_ao/key-management.c b/tools/testing/selftests/net/tcp_ao/key-management.c new file mode 100644 index 00000000000000..c48b4970ca17e0 --- /dev/null +++ b/tools/testing/selftests/net/tcp_ao/key-management.c @@ -0,0 +1,1180 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Author: Dmitry Safonov */ +#include +#include "../../../../include/linux/kernel.h" +#include "aolib.h" + +const size_t nr_packets = 20; +const size_t msg_len = 100; +const size_t quota = nr_packets * msg_len; +union tcp_addr wrong_addr; +#define SECOND_PASSWORD "at all times sincere friends of freedom have been rare" +#define fault(type) (inj == FAULT_ ## type) + +static const int test_vrf_ifindex = 200; +static const uint8_t test_vrf_tabid = 42; +static void setup_vrfs(void) +{ + int err; + + if (!kernel_config_has(KCONFIG_NET_VRF)) + return; + + err = add_vrf("ksft-vrf", test_vrf_tabid, test_vrf_ifindex, -1); + if (err) + test_error("Failed to add a VRF: %d", err); + + err = link_set_up("ksft-vrf"); + if (err) + test_error("Failed to bring up a VRF"); + + err = ip_route_add_vrf(veth_name, TEST_FAMILY, + this_ip_addr, this_ip_dest, test_vrf_tabid); + if (err) + test_error("Failed to add a route to VRF"); +} + + +static int prepare_sk(union tcp_addr *addr, uint8_t sndid, uint8_t rcvid) +{ + int sk = socket(test_family, SOCK_STREAM, IPPROTO_TCP); + + if (sk < 0) + test_error("socket()"); + + if (test_add_key(sk, DEFAULT_TEST_PASSWORD, this_ip_dest, + DEFAULT_TEST_PREFIX, 100, 100)) + test_error("test_add_key()"); + + if (addr && test_add_key(sk, SECOND_PASSWORD, *addr, + DEFAULT_TEST_PREFIX, sndid, rcvid)) + test_error("test_add_key()"); + + return sk; +} + +static int prepare_lsk(union tcp_addr *addr, uint8_t sndid, uint8_t rcvid) +{ + int sk = prepare_sk(addr, sndid, rcvid); + + if (listen(sk, 10)) + test_error("listen()"); + + return sk; +} + +static int test_del_key(int sk, uint8_t sndid, uint8_t rcvid, bool async, + int current_key, int rnext_key) +{ + struct tcp_ao_info_opt ao_info = {}; + struct tcp_ao_getsockopt key = {}; + struct tcp_ao_del del = {}; + sockaddr_af sockaddr; + int err; + + tcp_addr_to_sockaddr_in(&del.addr, &this_ip_dest, 0); + del.prefix = DEFAULT_TEST_PREFIX; + del.sndid = sndid; + del.rcvid = rcvid; + + if (current_key >= 0) { + del.set_current = 1; + del.current_key = (uint8_t)current_key; + } + if (rnext_key >= 0) { + del.set_rnext = 1; + del.rnext = (uint8_t)rnext_key; + } + + err = setsockopt(sk, IPPROTO_TCP, TCP_AO_DEL_KEY, &del, sizeof(del)); + if (err < 0) + return -errno; + + if (async) + return 0; + + tcp_addr_to_sockaddr_in(&sockaddr, &this_ip_dest, 0); + err = test_get_one_ao(sk, &key, &sockaddr, sizeof(sockaddr), + DEFAULT_TEST_PREFIX, sndid, rcvid); + if (!err) + return -EEXIST; + if (err != -E2BIG) + test_error("getsockopt()"); + if (current_key < 0 && rnext_key < 0) + return 0; + if (test_get_ao_info(sk, &ao_info)) + test_error("getsockopt(TCP_AO_INFO) failed"); + if (current_key >= 0 && ao_info.current_key != (uint8_t)current_key) + return -ENOTRECOVERABLE; + if (rnext_key >= 0 && ao_info.rnext != (uint8_t)rnext_key) + return -ENOTRECOVERABLE; + return 0; +} + +static void try_delete_key(char *tst_name, int sk, uint8_t sndid, uint8_t rcvid, + bool async, int current_key, int rnext_key, + fault_t inj) +{ + int err; + + err = test_del_key(sk, sndid, rcvid, async, current_key, rnext_key); + if ((err == -EBUSY && fault(BUSY)) || (err == -EINVAL && fault(CURRNEXT))) { + test_ok("%s: key deletion was prevented", tst_name); + return; + } + if (err && fault(FIXME)) { + test_xfail("%s: failed to delete the key %u:%u %d", + tst_name, sndid, rcvid, err); + return; + } + if (!err) { + if (fault(BUSY) || fault(CURRNEXT)) { + test_fail("%s: the key was deleted %u:%u %d", tst_name, + sndid, rcvid, err); + } else { + test_ok("%s: the key was deleted", tst_name); + } + return; + } + test_fail("%s: can't delete the key %u:%u %d", tst_name, sndid, rcvid, err); +} + +static int test_set_key(int sk, int current_keyid, int rnext_keyid) +{ + struct tcp_ao_info_opt ao_info = {}; + int err; + + if (current_keyid >= 0) { + ao_info.set_current = 1; + ao_info.current_key = (uint8_t)current_keyid; + } + if (rnext_keyid >= 0) { + ao_info.set_rnext = 1; + ao_info.rnext = (uint8_t)rnext_keyid; + } + + err = test_set_ao_info(sk, &ao_info); + if (err) + return err; + if (test_get_ao_info(sk, &ao_info)) + test_error("getsockopt(TCP_AO_INFO) failed"); + if (current_keyid >= 0 && ao_info.current_key != (uint8_t)current_keyid) + return -ENOTRECOVERABLE; + if (rnext_keyid >= 0 && ao_info.rnext != (uint8_t)rnext_keyid) + return -ENOTRECOVERABLE; + return 0; +} + +static int test_add_current_rnext_key(int sk, const char *key, uint8_t keyflags, + union tcp_addr in_addr, uint8_t prefix, + bool set_current, bool set_rnext, + uint8_t sndid, uint8_t rcvid) +{ + struct tcp_ao_add tmp = {}; + int err; + + err = test_prepare_key(&tmp, DEFAULT_TEST_ALGO, in_addr, + set_current, set_rnext, + prefix, 0, sndid, rcvid, 0, keyflags, + strlen(key), key); + if (err) + return err; + + + err = setsockopt(sk, IPPROTO_TCP, TCP_AO_ADD_KEY, &tmp, sizeof(tmp)); + if (err < 0) + return -errno; + + return test_verify_socket_key(sk, &tmp); +} + +static int __try_add_current_rnext_key(int sk, const char *key, uint8_t keyflags, + union tcp_addr in_addr, uint8_t prefix, + bool set_current, bool set_rnext, + uint8_t sndid, uint8_t rcvid) +{ + struct tcp_ao_info_opt ao_info = {}; + int err; + + err = test_add_current_rnext_key(sk, key, keyflags, in_addr, prefix, + set_current, set_rnext, sndid, rcvid); + if (err) + return err; + + if (test_get_ao_info(sk, &ao_info)) + test_error("getsockopt(TCP_AO_INFO) failed"); + if (set_current && ao_info.current_key != sndid) + return -ENOTRECOVERABLE; + if (set_rnext && ao_info.rnext != rcvid) + return -ENOTRECOVERABLE; + return 0; +} + +static void try_add_current_rnext_key(char *tst_name, int sk, const char *key, + uint8_t keyflags, + union tcp_addr in_addr, uint8_t prefix, + bool set_current, bool set_rnext, + uint8_t sndid, uint8_t rcvid, fault_t inj) +{ + int err; + + err = __try_add_current_rnext_key(sk, key, keyflags, in_addr, prefix, + set_current, set_rnext, sndid, rcvid); + if (!err && !fault(CURRNEXT)) { + test_ok("%s", tst_name); + return; + } + if (err == -EINVAL && fault(CURRNEXT)) { + test_ok("%s", tst_name); + return; + } + test_fail("%s", tst_name); +} + +static void check_closed_socket(void) +{ + int sk; + + sk = prepare_sk(&this_ip_dest, 200, 200); + try_delete_key("closed socket, delete a key", sk, 200, 200, 0, -1, -1, 0); + try_delete_key("closed socket, delete all keys", sk, 100, 100, 0, -1, -1, 0); + close(sk); + + sk = prepare_sk(&this_ip_dest, 200, 200); + if (test_set_key(sk, 100, 200)) + test_error("failed to set current/rnext keys"); + try_delete_key("closed socket, delete current key", sk, 100, 100, 0, -1, -1, FAULT_BUSY); + try_delete_key("closed socket, delete rnext key", sk, 200, 200, 0, -1, -1, FAULT_BUSY); + close(sk); + + sk = prepare_sk(&this_ip_dest, 200, 200); + if (test_add_key(sk, "Glory to heros!", this_ip_dest, + DEFAULT_TEST_PREFIX, 10, 11)) + test_error("test_add_key()"); + if (test_add_key(sk, "Glory to Ukraine!", this_ip_dest, + DEFAULT_TEST_PREFIX, 12, 13)) + test_error("test_add_key()"); + try_delete_key("closed socket, delete a key + set current/rnext", sk, 100, 100, 0, 10, 13, 0); + try_delete_key("closed socket, force-delete current key", sk, 10, 11, 0, 200, -1, 0); + try_delete_key("closed socket, force-delete rnext key", sk, 12, 13, 0, -1, 200, 0); + try_delete_key("closed socket, delete current+rnext key", sk, 200, 200, 0, -1, -1, FAULT_BUSY); + close(sk); + + sk = prepare_sk(&this_ip_dest, 200, 200); + if (test_set_key(sk, 100, 200)) + test_error("failed to set current/rnext keys"); + try_add_current_rnext_key("closed socket, add + change current key", + sk, "Laaaa! Lalala-la-la-lalala...", 0, + this_ip_dest, DEFAULT_TEST_PREFIX, + true, false, 10, 20, 0); + try_add_current_rnext_key("closed socket, add + change rnext key", + sk, "Laaaa! Lalala-la-la-lalala...", 0, + this_ip_dest, DEFAULT_TEST_PREFIX, + false, true, 20, 10, 0); + close(sk); +} + +static void assert_no_current_rnext(const char *tst_msg, int sk) +{ + struct tcp_ao_info_opt ao_info = {}; + + if (test_get_ao_info(sk, &ao_info)) + test_error("getsockopt(TCP_AO_INFO) failed"); + + errno = 0; + if (ao_info.set_current || ao_info.set_rnext) { + test_xfail("%s: the socket has current/rnext keys: %d:%d", + tst_msg, + (ao_info.set_current) ? ao_info.current_key : -1, + (ao_info.set_rnext) ? ao_info.rnext : -1); + } else { + test_ok("%s: the socket has no current/rnext keys", tst_msg); + } +} + +static void assert_no_tcp_repair(void) +{ + struct tcp_ao_repair ao_img = {}; + socklen_t len = sizeof(ao_img); + int sk, err; + + sk = prepare_sk(&this_ip_dest, 200, 200); + test_enable_repair(sk); + if (listen(sk, 10)) + test_error("listen()"); + errno = 0; + err = getsockopt(sk, SOL_TCP, TCP_AO_REPAIR, &ao_img, &len); + if (err && errno == EPERM) + test_ok("listen socket, getsockopt(TCP_AO_REPAIR) is restricted"); + else + test_fail("listen socket, getsockopt(TCP_AO_REPAIR) works"); + errno = 0; + err = setsockopt(sk, SOL_TCP, TCP_AO_REPAIR, &ao_img, sizeof(ao_img)); + if (err && errno == EPERM) + test_ok("listen socket, setsockopt(TCP_AO_REPAIR) is restricted"); + else + test_fail("listen socket, setsockopt(TCP_AO_REPAIR) works"); + close(sk); +} + +static void check_listen_socket(void) +{ + int sk, err; + + sk = prepare_lsk(&this_ip_dest, 200, 200); + try_delete_key("listen socket, delete a key", sk, 200, 200, 0, -1, -1, 0); + try_delete_key("listen socket, delete all keys", sk, 100, 100, 0, -1, -1, 0); + close(sk); + + sk = prepare_lsk(&this_ip_dest, 200, 200); + err = test_set_key(sk, 100, -1); + if (err == -EINVAL) + test_ok("listen socket, setting current key not allowed"); + else + test_fail("listen socket, set current key"); + err = test_set_key(sk, -1, 200); + if (err == -EINVAL) + test_ok("listen socket, setting rnext key not allowed"); + else + test_fail("listen socket, set rnext key"); + close(sk); + + sk = prepare_sk(&this_ip_dest, 200, 200); + if (test_set_key(sk, 100, 200)) + test_error("failed to set current/rnext keys"); + if (listen(sk, 10)) + test_error("listen()"); + assert_no_current_rnext("listen() after current/rnext keys set", sk); + try_delete_key("listen socket, delete current key from before listen()", sk, 100, 100, 0, -1, -1, FAULT_FIXME); + try_delete_key("listen socket, delete rnext key from before listen()", sk, 200, 200, 0, -1, -1, FAULT_FIXME); + close(sk); + + assert_no_tcp_repair(); + + sk = prepare_lsk(&this_ip_dest, 200, 200); + if (test_add_key(sk, "Glory to heros!", this_ip_dest, + DEFAULT_TEST_PREFIX, 10, 11)) + test_error("test_add_key()"); + if (test_add_key(sk, "Glory to Ukraine!", this_ip_dest, + DEFAULT_TEST_PREFIX, 12, 13)) + test_error("test_add_key()"); + try_delete_key("listen socket, delete a key + set current/rnext", sk, + 100, 100, 0, 10, 13, FAULT_CURRNEXT); + try_delete_key("listen socket, force-delete current key", sk, + 10, 11, 0, 200, -1, FAULT_CURRNEXT); + try_delete_key("listen socket, force-delete rnext key", sk, + 12, 13, 0, -1, 200, FAULT_CURRNEXT); + try_delete_key("listen socket, delete a key", sk, + 200, 200, 0, -1, -1, 0); + close(sk); + + sk = prepare_lsk(&this_ip_dest, 200, 200); + try_add_current_rnext_key("listen socket, add + change current key", + sk, "Laaaa! Lalala-la-la-lalala...", 0, + this_ip_dest, DEFAULT_TEST_PREFIX, + true, false, 10, 20, FAULT_CURRNEXT); + try_add_current_rnext_key("listen socket, add + change rnext key", + sk, "Laaaa! Lalala-la-la-lalala...", 0, + this_ip_dest, DEFAULT_TEST_PREFIX, + false, true, 20, 10, FAULT_CURRNEXT); + close(sk); +} + +static const char *fips_fpath = "/proc/sys/crypto/fips_enabled"; +static bool is_fips_enabled(void) +{ + static int fips_checked = -1; + FILE *fenabled; + int enabled; + + if (fips_checked >= 0) + return !!fips_checked; + if (access(fips_fpath, R_OK)) { + if (errno != ENOENT) + test_error("Can't open %s", fips_fpath); + fips_checked = 0; + return false; + } + fenabled = fopen(fips_fpath, "r"); + if (!fenabled) + test_error("Can't open %s", fips_fpath); + if (fscanf(fenabled, "%d", &enabled) != 1) + test_error("Can't read from %s", fips_fpath); + fclose(fenabled); + fips_checked = !!enabled; + return !!fips_checked; +} + +struct test_key { + char password[TCP_AO_MAXKEYLEN]; + const char *alg; + unsigned int len; + uint8_t client_keyid; + uint8_t server_keyid; + uint8_t maclen; + uint8_t matches_client : 1, + matches_server : 1, + matches_vrf : 1, + is_current : 1, + is_rnext : 1, + used_on_handshake : 1, + used_after_accept : 1, + used_on_client : 1; +}; + +struct key_collection { + unsigned int nr_keys; + struct test_key *keys; +}; + +static struct key_collection collection; + +#define TEST_MAX_MACLEN 16 +const char *test_algos[] = { + "cmac(aes128)", + "hmac(sha1)", "hmac(sha512)", "hmac(sha384)", "hmac(sha256)", + "hmac(sha224)", "hmac(sha3-512)", + /* only if !CONFIG_FIPS */ +#define TEST_NON_FIPS_ALGOS 2 + "hmac(rmd160)", "hmac(md5)" +}; +const unsigned int test_maclens[] = { 1, 4, 12, 16 }; +#define MACLEN_SHIFT 2 +#define ALGOS_SHIFT 4 + +static unsigned int make_mask(unsigned int shift, unsigned int prev_shift) +{ + unsigned int ret = BIT(shift) - 1; + + return ret << prev_shift; +} + +static void init_key_in_collection(unsigned int index, bool randomized) +{ + struct test_key *key = &collection.keys[index]; + unsigned int algos_nr, algos_index; + + /* Same for randomized and non-randomized test flows */ + key->client_keyid = index; + key->server_keyid = 127 + index; + key->matches_client = 1; + key->matches_server = 1; + key->matches_vrf = 1; + /* not really even random, but good enough for a test */ + key->len = rand() % (TCP_AO_MAXKEYLEN - TEST_TCP_AO_MINKEYLEN); + key->len += TEST_TCP_AO_MINKEYLEN; + randomize_buffer(key->password, key->len); + + if (randomized) { + key->maclen = (rand() % TEST_MAX_MACLEN) + 1; + algos_index = rand(); + } else { + unsigned int shift = MACLEN_SHIFT; + + key->maclen = test_maclens[index & make_mask(shift, 0)]; + algos_index = index & make_mask(ALGOS_SHIFT, shift); + } + algos_nr = ARRAY_SIZE(test_algos); + if (is_fips_enabled()) + algos_nr -= TEST_NON_FIPS_ALGOS; + key->alg = test_algos[algos_index % algos_nr]; +} + +static int init_default_key_collection(unsigned int nr_keys, bool randomized) +{ + size_t key_sz = sizeof(collection.keys[0]); + + if (!nr_keys) { + free(collection.keys); + collection.keys = NULL; + return 0; + } + + /* + * All keys have uniq sndid/rcvid and sndid != rcvid in order to + * check for any bugs/issues for different keyids, visible to both + * peers. Keyid == 254 is unused. + */ + if (nr_keys > 127) + test_error("Test requires too many keys, correct the source"); + + collection.keys = reallocarray(collection.keys, nr_keys, key_sz); + if (!collection.keys) + return -ENOMEM; + + memset(collection.keys, 0, nr_keys * key_sz); + collection.nr_keys = nr_keys; + while (nr_keys--) + init_key_in_collection(nr_keys, randomized); + + return 0; +} + +static void test_key_error(const char *msg, struct test_key *key) +{ + test_error("%s: key: { %s, %u:%u, %u, %u:%u:%u:%u:%u (%u)}", + msg, key->alg, key->client_keyid, key->server_keyid, + key->maclen, key->matches_client, key->matches_server, + key->matches_vrf, key->is_current, key->is_rnext, key->len); +} + +static int test_add_key_cr(int sk, const char *pwd, unsigned int pwd_len, + union tcp_addr addr, uint8_t vrf, + uint8_t sndid, uint8_t rcvid, + uint8_t maclen, const char *alg, + bool set_current, bool set_rnext) +{ + struct tcp_ao_add tmp = {}; + uint8_t keyflags = 0; + int err; + + if (!alg) + alg = DEFAULT_TEST_ALGO; + + if (vrf) + keyflags |= TCP_AO_KEYF_IFINDEX; + err = test_prepare_key(&tmp, alg, addr, set_current, set_rnext, + DEFAULT_TEST_PREFIX, vrf, sndid, rcvid, maclen, + keyflags, pwd_len, pwd); + if (err) + return err; + + err = setsockopt(sk, IPPROTO_TCP, TCP_AO_ADD_KEY, &tmp, sizeof(tmp)); + if (err < 0) + return -errno; + + return test_verify_socket_key(sk, &tmp); +} + +static void verify_current_rnext(const char *tst, int sk, + int current_keyid, int rnext_keyid) +{ + struct tcp_ao_info_opt ao_info = {}; + + if (test_get_ao_info(sk, &ao_info)) + test_error("getsockopt(TCP_AO_INFO) failed"); + + errno = 0; + if (current_keyid >= 0) { + if (!ao_info.set_current) + test_fail("%s: the socket doesn't have current key", tst); + else if (ao_info.current_key != current_keyid) + test_fail("%s: current key is not the expected one %d != %u", + tst, current_keyid, ao_info.current_key); + else + test_ok("%s: current key %u as expected", + tst, ao_info.current_key); + } + if (rnext_keyid >= 0) { + if (!ao_info.set_rnext) + test_fail("%s: the socket doesn't have rnext key", tst); + else if (ao_info.rnext != rnext_keyid) + test_fail("%s: rnext key is not the expected one %d != %u", + tst, rnext_keyid, ao_info.rnext); + else + test_ok("%s: rnext key %u as expected", tst, ao_info.rnext); + } +} + + +static int key_collection_socket(bool server, unsigned int port) +{ + unsigned int i; + int sk; + + if (server) + sk = test_listen_socket(this_ip_addr, port, 1); + else + sk = socket(test_family, SOCK_STREAM, IPPROTO_TCP); + if (sk < 0) + test_error("socket()"); + + for (i = 0; i < collection.nr_keys; i++) { + struct test_key *key = &collection.keys[i]; + union tcp_addr *addr = &wrong_addr; + uint8_t sndid, rcvid, vrf; + bool set_current = false, set_rnext = false; + + if (key->matches_vrf) + vrf = 0; + else + vrf = test_vrf_ifindex; + if (server) { + if (key->matches_client) + addr = &this_ip_dest; + sndid = key->server_keyid; + rcvid = key->client_keyid; + } else { + if (key->matches_server) + addr = &this_ip_dest; + sndid = key->client_keyid; + rcvid = key->server_keyid; + set_current = key->is_current; + set_rnext = key->is_rnext; + } + + if (test_add_key_cr(sk, key->password, key->len, + *addr, vrf, sndid, rcvid, key->maclen, + key->alg, set_current, set_rnext)) + test_key_error("setsockopt(TCP_AO_ADD_KEY)", key); + if (set_current || set_rnext) + key->used_on_handshake = 1; +#ifdef DEBUG + test_print("%s [%u/%u] key: { %s, %u:%u, %u, %u:%u:%u:%u (%u)}", + server ? "server" : "client", i, collection.nr_keys, + key->alg, rcvid, sndid, key->maclen, + key->matches_client, key->matches_server, + key->is_current, key->is_rnext, key->len); +#endif + } + return sk; +} + +static void verify_counters(const char *tst_name, bool is_listen_sk, bool server, + struct tcp_ao_counters *a, struct tcp_ao_counters *b) +{ + unsigned int i; + + __test_tcp_ao_counters_cmp(tst_name, a, b, TEST_CNT_GOOD); + + for (i = 0; i < collection.nr_keys; i++) { + struct test_key *key = &collection.keys[i]; + uint8_t sndid, rcvid; + bool was_used; + + if (server) { + sndid = key->server_keyid; + rcvid = key->client_keyid; + if (is_listen_sk) + was_used = key->used_on_handshake; + else + was_used = key->used_after_accept; + } else { + sndid = key->client_keyid; + rcvid = key->server_keyid; + was_used = key->used_on_client; + } + + test_tcp_ao_key_counters_cmp(tst_name, a, b, was_used, + sndid, rcvid); + } + test_tcp_ao_counters_free(a); + test_tcp_ao_counters_free(b); + test_ok("%s: passed counters checks", tst_name); +} + +static struct tcp_ao_getsockopt *lookup_key(struct tcp_ao_getsockopt *buf, + size_t len, int sndid, int rcvid) +{ + size_t i; + + for (i = 0; i < len; i++) { + if (sndid >= 0 && buf[i].sndid != sndid) + continue; + if (rcvid >= 0 && buf[i].rcvid != rcvid) + continue; + return &buf[i]; + } + return NULL; +} + +static void verify_keys(const char *tst_name, int sk, + bool is_listen_sk, bool server) +{ + socklen_t len = sizeof(struct tcp_ao_getsockopt); + struct tcp_ao_getsockopt *keys; + bool passed_test = true; + unsigned int i; + + keys = calloc(collection.nr_keys, len); + if (!keys) + test_error("calloc()"); + + keys->nkeys = collection.nr_keys; + keys->get_all = 1; + + if (getsockopt(sk, IPPROTO_TCP, TCP_AO_GET_KEYS, keys, &len)) { + free(keys); + test_error("getsockopt(TCP_AO_GET_KEYS)"); + } + + for (i = 0; i < collection.nr_keys; i++) { + struct test_key *key = &collection.keys[i]; + struct tcp_ao_getsockopt *dump_key; + bool is_kdf_aes_128_cmac = false; + bool is_cmac_aes = false; + uint8_t sndid, rcvid; + bool matches = false; + + if (server) { + if (key->matches_client) + matches = true; + sndid = key->server_keyid; + rcvid = key->client_keyid; + } else { + if (key->matches_server) + matches = true; + sndid = key->client_keyid; + rcvid = key->server_keyid; + } + if (!key->matches_vrf) + matches = false; + /* no keys get removed on the original listener socket */ + if (is_listen_sk) + matches = true; + + dump_key = lookup_key(keys, keys->nkeys, sndid, rcvid); + if (matches != !!dump_key) { + test_fail("%s: key %u:%u %s%s on the socket", + tst_name, sndid, rcvid, + key->matches_vrf ? "" : "[vrf] ", + matches ? "disappeared" : "yet present"); + passed_test = false; + goto out; + } + if (!dump_key) + continue; + + if (!strcmp("cmac(aes128)", key->alg)) { + is_kdf_aes_128_cmac = (key->len != 16); + is_cmac_aes = true; + } + + if (is_cmac_aes) { + if (strcmp(dump_key->alg_name, "cmac(aes)")) { + test_fail("%s: key %u:%u cmac(aes) has unexpected alg %s", + tst_name, sndid, rcvid, + dump_key->alg_name); + passed_test = false; + continue; + } + } else if (strcmp(dump_key->alg_name, key->alg)) { + test_fail("%s: key %u:%u has unexpected alg %s != %s", + tst_name, sndid, rcvid, + dump_key->alg_name, key->alg); + passed_test = false; + continue; + } + if (is_kdf_aes_128_cmac) { + if (dump_key->keylen != 16) { + test_fail("%s: key %u:%u cmac(aes128) has unexpected len %u", + tst_name, sndid, rcvid, + dump_key->keylen); + continue; + } + } else if (dump_key->keylen != key->len) { + test_fail("%s: key %u:%u changed password len %u != %u", + tst_name, sndid, rcvid, + dump_key->keylen, key->len); + passed_test = false; + continue; + } + if (!is_kdf_aes_128_cmac && + memcmp(dump_key->key, key->password, key->len)) { + test_fail("%s: key %u:%u has different password", + tst_name, sndid, rcvid); + passed_test = false; + continue; + } + if (dump_key->maclen != key->maclen) { + test_fail("%s: key %u:%u changed maclen %u != %u", + tst_name, sndid, rcvid, + dump_key->maclen, key->maclen); + passed_test = false; + continue; + } + } + + if (passed_test) + test_ok("%s: The socket keys are consistent with the expectations", + tst_name); +out: + free(keys); +} + +static int start_server(const char *tst_name, unsigned int port, size_t quota, + struct tcp_ao_counters *begin, + unsigned int current_index, unsigned int rnext_index) +{ + struct tcp_ao_counters lsk_c1, lsk_c2; + ssize_t bytes; + int sk, lsk; + + synchronize_threads(); /* 1: key collection initialized */ + lsk = key_collection_socket(true, port); + if (test_get_tcp_ao_counters(lsk, &lsk_c1)) + test_error("test_get_tcp_ao_counters()"); + synchronize_threads(); /* 2: MKTs added => connect() */ + if (test_wait_fd(lsk, TEST_TIMEOUT_SEC, 0)) + test_error("test_wait_fd()"); + + sk = accept(lsk, NULL, NULL); + if (sk < 0) + test_error("accept()"); + if (test_get_tcp_ao_counters(sk, begin)) + test_error("test_get_tcp_ao_counters()"); + + synchronize_threads(); /* 3: accepted => send data */ + if (test_get_tcp_ao_counters(lsk, &lsk_c2)) + test_error("test_get_tcp_ao_counters()"); + verify_keys(tst_name, lsk, true, true); + close(lsk); + + bytes = test_server_run(sk, quota, TEST_TIMEOUT_SEC); + if (bytes != quota) + test_fail("%s: server served: %zd", tst_name, bytes); + else + test_ok("%s: server alive", tst_name); + + verify_counters(tst_name, true, true, &lsk_c1, &lsk_c2); + + return sk; +} + +static void end_server(const char *tst_name, int sk, + struct tcp_ao_counters *begin) +{ + struct tcp_ao_counters end; + + if (test_get_tcp_ao_counters(sk, &end)) + test_error("test_get_tcp_ao_counters()"); + verify_keys(tst_name, sk, false, true); + + synchronize_threads(); /* 4: verified => closed */ + close(sk); + + verify_counters(tst_name, true, false, begin, &end); + synchronize_threads(); /* 5: counters */ +} + +static void try_server_run(const char *tst_name, unsigned int port, size_t quota, + unsigned int current_index, unsigned int rnext_index) +{ + struct tcp_ao_counters tmp; + int sk; + + sk = start_server(tst_name, port, quota, &tmp, + current_index, rnext_index); + end_server(tst_name, sk, &tmp); +} + +static void server_rotations(const char *tst_name, unsigned int port, + size_t quota, unsigned int rotations, + unsigned int current_index, unsigned int rnext_index) +{ + struct tcp_ao_counters tmp; + unsigned int i; + int sk; + + sk = start_server(tst_name, port, quota, &tmp, + current_index, rnext_index); + + for (i = current_index + 1; rotations > 0; i++, rotations--) { + ssize_t bytes; + + if (i >= collection.nr_keys) + i = 0; + bytes = test_server_run(sk, quota, TEST_TIMEOUT_SEC); + if (bytes != quota) { + test_fail("%s: server served: %zd", tst_name, bytes); + return; + } + verify_current_rnext(tst_name, sk, + collection.keys[i].server_keyid, -1); + synchronize_threads(); /* verify current/rnext */ + } + end_server(tst_name, sk, &tmp); +} + +static int run_client(const char *tst_name, unsigned int port, + unsigned int nr_keys, int current_index, int rnext_index, + struct tcp_ao_counters *before, + const size_t msg_sz, const size_t msg_nr) +{ + int sk; + + synchronize_threads(); /* 1: key collection initialized */ + sk = key_collection_socket(false, port); + + if (current_index >= 0 || rnext_index >= 0) { + int sndid = -1, rcvid = -1; + + if (current_index >= 0) + sndid = collection.keys[current_index].client_keyid; + if (rnext_index >= 0) + rcvid = collection.keys[rnext_index].server_keyid; + if (test_set_key(sk, sndid, rcvid)) + test_error("failed to set current/rnext keys"); + } + if (before && test_get_tcp_ao_counters(sk, before)) + test_error("test_get_tcp_ao_counters()"); + + synchronize_threads(); /* 2: MKTs added => connect() */ + if (test_connect_socket(sk, this_ip_dest, port++) <= 0) + test_error("failed to connect()"); + if (current_index < 0) + current_index = nr_keys - 1; + if (rnext_index < 0) + rnext_index = nr_keys - 1; + collection.keys[current_index].used_on_handshake = 1; + collection.keys[rnext_index].used_after_accept = 1; + collection.keys[rnext_index].used_on_client = 1; + + synchronize_threads(); /* 3: accepted => send data */ + if (test_client_verify(sk, msg_sz, msg_nr, TEST_TIMEOUT_SEC)) { + test_fail("verify failed"); + close(sk); + if (before) + test_tcp_ao_counters_free(before); + return -1; + } + + return sk; +} + +static int start_client(const char *tst_name, unsigned int port, + unsigned int nr_keys, int current_index, int rnext_index, + struct tcp_ao_counters *before, + const size_t msg_sz, const size_t msg_nr) +{ + if (init_default_key_collection(nr_keys, true)) + test_error("Failed to init the key collection"); + + return run_client(tst_name, port, nr_keys, current_index, + rnext_index, before, msg_sz, msg_nr); +} + +static void end_client(const char *tst_name, int sk, unsigned int nr_keys, + int current_index, int rnext_index, + struct tcp_ao_counters *start) +{ + struct tcp_ao_counters end; + + /* Some application may become dependent on this kernel choice */ + if (current_index < 0) + current_index = nr_keys - 1; + if (rnext_index < 0) + rnext_index = nr_keys - 1; + verify_current_rnext(tst_name, sk, + collection.keys[current_index].client_keyid, + collection.keys[rnext_index].server_keyid); + if (start && test_get_tcp_ao_counters(sk, &end)) + test_error("test_get_tcp_ao_counters()"); + verify_keys(tst_name, sk, false, false); + synchronize_threads(); /* 4: verify => closed */ + close(sk); + if (start) + verify_counters(tst_name, false, false, start, &end); + synchronize_threads(); /* 5: counters */ +} + +static void try_unmatched_keys(int sk, int *rnext_index) +{ + struct test_key *key; + unsigned int i = 0; + int err; + + do { + key = &collection.keys[i]; + if (!key->matches_server) + break; + } while (++i < collection.nr_keys); + if (key->matches_server) + test_error("all keys on client match the server"); + + err = test_add_key_cr(sk, key->password, key->len, wrong_addr, + 0, key->client_keyid, key->server_keyid, + key->maclen, key->alg, 0, 0); + if (!err) { + test_fail("Added a key with non-matching ip-address for established sk"); + return; + } + if (err == -EINVAL) + test_ok("Can't add a key with non-matching ip-address for established sk"); + else + test_error("Failed to add a key"); + + err = test_add_key_cr(sk, key->password, key->len, this_ip_dest, + test_vrf_ifindex, + key->client_keyid, key->server_keyid, + key->maclen, key->alg, 0, 0); + if (!err) { + test_fail("Added a key with non-matching VRF for established sk"); + return; + } + if (err == -EINVAL) + test_ok("Can't add a key with non-matching VRF for established sk"); + else + test_error("Failed to add a key"); + + for (i = 0; i < collection.nr_keys; i++) { + key = &collection.keys[i]; + if (!key->matches_client) + break; + } + if (key->matches_client) + test_error("all keys on server match the client"); + if (test_set_key(sk, -1, key->server_keyid)) + test_error("Can't change the current key"); + if (test_client_verify(sk, msg_len, nr_packets, TEST_TIMEOUT_SEC)) + test_fail("verify failed"); + *rnext_index = i; +} + +static int client_non_matching(const char *tst_name, unsigned int port, + unsigned int nr_keys, + int current_index, int rnext_index, + const size_t msg_sz, const size_t msg_nr) +{ + unsigned int i; + + if (init_default_key_collection(nr_keys, true)) + test_error("Failed to init the key collection"); + + for (i = 0; i < nr_keys; i++) { + /* key (0, 0) matches */ + collection.keys[i].matches_client = !!((i + 3) % 4); + collection.keys[i].matches_server = !!((i + 2) % 4); + if (kernel_config_has(KCONFIG_NET_VRF)) + collection.keys[i].matches_vrf = !!((i + 1) % 4); + } + + return run_client(tst_name, port, nr_keys, current_index, + rnext_index, NULL, msg_sz, msg_nr); +} + +static void check_current_back(const char *tst_name, unsigned int port, + unsigned int nr_keys, + unsigned int current_index, unsigned int rnext_index, + unsigned int rotate_to_index) +{ + struct tcp_ao_counters tmp; + int sk; + + sk = start_client(tst_name, port, nr_keys, current_index, rnext_index, + &tmp, msg_len, nr_packets); + if (sk < 0) + return; + if (test_set_key(sk, collection.keys[rotate_to_index].client_keyid, -1)) + test_error("Can't change the current key"); + if (test_client_verify(sk, msg_len, nr_packets, TEST_TIMEOUT_SEC)) + test_fail("verify failed"); + collection.keys[rotate_to_index].used_after_accept = 1; + + end_client(tst_name, sk, nr_keys, current_index, rnext_index, &tmp); +} + +static void roll_over_keys(const char *tst_name, unsigned int port, + unsigned int nr_keys, unsigned int rotations, + unsigned int current_index, unsigned int rnext_index) +{ + struct tcp_ao_counters tmp; + unsigned int i; + int sk; + + sk = start_client(tst_name, port, nr_keys, current_index, rnext_index, + &tmp, msg_len, nr_packets); + if (sk < 0) + return; + for (i = rnext_index + 1; rotations > 0; i++, rotations--) { + if (i >= collection.nr_keys) + i = 0; + if (test_set_key(sk, -1, collection.keys[i].server_keyid)) + test_error("Can't change the Rnext key"); + if (test_client_verify(sk, msg_len, nr_packets, TEST_TIMEOUT_SEC)) { + test_fail("verify failed"); + close(sk); + test_tcp_ao_counters_free(&tmp); + return; + } + verify_current_rnext(tst_name, sk, -1, + collection.keys[i].server_keyid); + collection.keys[i].used_on_client = 1; + synchronize_threads(); /* verify current/rnext */ + } + end_client(tst_name, sk, nr_keys, current_index, rnext_index, &tmp); +} + +static void try_client_run(const char *tst_name, unsigned int port, + unsigned int nr_keys, int current_index, int rnext_index) +{ + struct tcp_ao_counters tmp; + int sk; + + sk = start_client(tst_name, port, nr_keys, current_index, rnext_index, + &tmp, msg_len, nr_packets); + if (sk < 0) + return; + end_client(tst_name, sk, nr_keys, current_index, rnext_index, &tmp); +} + +static void try_client_match(const char *tst_name, unsigned int port, + unsigned int nr_keys, + int current_index, int rnext_index) +{ + int sk; + + sk = client_non_matching(tst_name, port, nr_keys, current_index, + rnext_index, msg_len, nr_packets); + if (sk < 0) + return; + try_unmatched_keys(sk, &rnext_index); + end_client(tst_name, sk, nr_keys, current_index, rnext_index, NULL); +} + +static void *server_fn(void *arg) +{ + unsigned int port = test_server_port; + + setup_vrfs(); + try_server_run("server: Check current/rnext keys unset before connect()", + port++, quota, 19, 19); + try_server_run("server: Check current/rnext keys set before connect()", + port++, quota, 10, 10); + try_server_run("server: Check current != rnext keys set before connect()", + port++, quota, 5, 10); + try_server_run("server: Check current flapping back on peer's RnextKey request", + port++, quota * 2, 5, 10); + server_rotations("server: Rotate over all different keys", port++, + quota, 20, 0, 0); + try_server_run("server: Check accept() => established key matching", + port++, quota * 2, 0, 0); + + synchronize_threads(); /* don't race to exit: client exits */ + return NULL; +} + +static void check_established_socket(void) +{ + unsigned int port = test_server_port; + + setup_vrfs(); + try_client_run("client: Check current/rnext keys unset before connect()", + port++, 20, -1, -1); + try_client_run("client: Check current/rnext keys set before connect()", + port++, 20, 10, 10); + try_client_run("client: Check current != rnext keys set before connect()", + port++, 20, 10, 5); + check_current_back("client: Check current flapping back on peer's RnextKey request", + port++, 20, 10, 5, 2); + roll_over_keys("client: Rotate over all different keys", port++, + 20, 20, 0, 0); + try_client_match("client: Check connect() => established key matching", + port++, 20, 0, 0); +} + +static void *client_fn(void *arg) +{ + if (inet_pton(TEST_FAMILY, TEST_WRONG_IP, &wrong_addr) != 1) + test_error("Can't convert ip address %s", TEST_WRONG_IP); + check_closed_socket(); + check_listen_socket(); + check_established_socket(); + return NULL; +} + +int main(int argc, char *argv[]) +{ + test_init(120, server_fn, client_fn); + return 0; +} diff --git a/tools/testing/selftests/net/tcp_ao/lib/aolib.h b/tools/testing/selftests/net/tcp_ao/lib/aolib.h new file mode 100644 index 00000000000000..fbc7f61118159e --- /dev/null +++ b/tools/testing/selftests/net/tcp_ao/lib/aolib.h @@ -0,0 +1,605 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * TCP-AO selftest library. Provides helpers to unshare network + * namespaces, create veth, assign ip addresses, set routes, + * manipulate socket options, read network counter and etc. + * Author: Dmitry Safonov + */ +#ifndef _AOLIB_H_ +#define _AOLIB_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../../../../include/linux/stringify.h" +#include "../../../../../include/linux/bits.h" + +#ifndef SOL_TCP +/* can't include as including */ +# define SOL_TCP 6 /* TCP level */ +#endif + +/* Working around ksft, see the comment in lib/setup.c */ +extern void __test_msg(const char *buf); +extern void __test_ok(const char *buf); +extern void __test_fail(const char *buf); +extern void __test_xfail(const char *buf); +extern void __test_error(const char *buf); +extern void __test_skip(const char *buf); + +__attribute__((__format__(__printf__, 2, 3))) +static inline void __test_print(void (*fn)(const char *), const char *fmt, ...) +{ +#define TEST_MSG_BUFFER_SIZE 4096 + char buf[TEST_MSG_BUFFER_SIZE]; + va_list arg; + + va_start(arg, fmt); + vsnprintf(buf, sizeof(buf), fmt, arg); + va_end(arg); + fn(buf); +} + +#define test_print(fmt, ...) \ + __test_print(__test_msg, "%ld[%s:%u] " fmt "\n", \ + syscall(SYS_gettid), \ + __FILE__, __LINE__, ##__VA_ARGS__) + +#define test_ok(fmt, ...) \ + __test_print(__test_ok, fmt "\n", ##__VA_ARGS__) +#define test_skip(fmt, ...) \ + __test_print(__test_skip, fmt "\n", ##__VA_ARGS__) +#define test_xfail(fmt, ...) \ + __test_print(__test_xfail, fmt "\n", ##__VA_ARGS__) + +#define test_fail(fmt, ...) \ +do { \ + if (errno) \ + __test_print(__test_fail, fmt ": %m\n", ##__VA_ARGS__); \ + else \ + __test_print(__test_fail, fmt "\n", ##__VA_ARGS__); \ + test_failed(); \ +} while (0) + +#define KSFT_FAIL 1 +#define test_error(fmt, ...) \ +do { \ + if (errno) \ + __test_print(__test_error, "%ld[%s:%u] " fmt ": %m\n", \ + syscall(SYS_gettid), __FILE__, __LINE__, \ + ##__VA_ARGS__); \ + else \ + __test_print(__test_error, "%ld[%s:%u] " fmt "\n", \ + syscall(SYS_gettid), __FILE__, __LINE__, \ + ##__VA_ARGS__); \ + exit(KSFT_FAIL); \ +} while (0) + +enum test_fault { + FAULT_TIMEOUT = 1, + FAULT_KEYREJECT, + FAULT_PREINSTALL_AO, + FAULT_PREINSTALL_MD5, + FAULT_POSTINSTALL, + FAULT_BUSY, + FAULT_CURRNEXT, + FAULT_FIXME, +}; +typedef enum test_fault fault_t; + +enum test_needs_kconfig { + KCONFIG_NET_NS = 0, /* required */ + KCONFIG_VETH, /* required */ + KCONFIG_TCP_AO, /* required */ + KCONFIG_TCP_MD5, /* optional, for TCP-MD5 features */ + KCONFIG_NET_VRF, /* optional, for L3/VRF testing */ + __KCONFIG_LAST__ +}; +extern bool kernel_config_has(enum test_needs_kconfig k); +extern const char *tests_skip_reason[__KCONFIG_LAST__]; +static inline bool should_skip_test(const char *tst_name, + enum test_needs_kconfig k) +{ + if (kernel_config_has(k)) + return false; + test_skip("%s: %s", tst_name, tests_skip_reason[k]); + return true; +} + +union tcp_addr { + struct in_addr a4; + struct in6_addr a6; +}; + +typedef void *(*thread_fn)(void *); +extern void test_failed(void); +extern void __test_init(unsigned int ntests, int family, unsigned int prefix, + union tcp_addr addr1, union tcp_addr addr2, + thread_fn peer1, thread_fn peer2); + +static inline void test_init2(unsigned int ntests, + thread_fn peer1, thread_fn peer2, + int family, unsigned int prefix, + const char *addr1, const char *addr2) +{ + union tcp_addr taddr1, taddr2; + + if (inet_pton(family, addr1, &taddr1) != 1) + test_error("Can't convert ip address %s", addr1); + if (inet_pton(family, addr2, &taddr2) != 1) + test_error("Can't convert ip address %s", addr2); + + __test_init(ntests, family, prefix, taddr1, taddr2, peer1, peer2); +} +extern void test_add_destructor(void (*d)(void)); + +/* To adjust optmem socket limit, approximately estimate a number, + * that is bigger than sizeof(struct tcp_ao_key). + */ +#define KERNEL_TCP_AO_KEY_SZ_ROUND_UP 300 + +extern void test_set_optmem(size_t value); +extern size_t test_get_optmem(void); + +extern const struct sockaddr_in6 addr_any6; +extern const struct sockaddr_in addr_any4; + +#ifdef IPV6_TEST +# define __TEST_CLIENT_IP(n) ("2001:db8:" __stringify(n) "::1") +# define TEST_CLIENT_IP __TEST_CLIENT_IP(1) +# define TEST_WRONG_IP "2001:db8:253::1" +# define TEST_SERVER_IP "2001:db8:254::1" +# define TEST_NETWORK "2001::" +# define TEST_PREFIX 128 +# define TEST_FAMILY AF_INET6 +# define SOCKADDR_ANY addr_any6 +# define sockaddr_af struct sockaddr_in6 +#else +# define __TEST_CLIENT_IP(n) ("10.0." __stringify(n) ".1") +# define TEST_CLIENT_IP __TEST_CLIENT_IP(1) +# define TEST_WRONG_IP "10.0.253.1" +# define TEST_SERVER_IP "10.0.254.1" +# define TEST_NETWORK "10.0.0.0" +# define TEST_PREFIX 32 +# define TEST_FAMILY AF_INET +# define SOCKADDR_ANY addr_any4 +# define sockaddr_af struct sockaddr_in +#endif + +static inline union tcp_addr gen_tcp_addr(union tcp_addr net, size_t n) +{ + union tcp_addr ret = net; + +#ifdef IPV6_TEST + ret.a6.s6_addr32[3] = htonl(n & (BIT(32) - 1)); + ret.a6.s6_addr32[2] = htonl((n >> 32) & (BIT(32) - 1)); +#else + ret.a4.s_addr = htonl(ntohl(net.a4.s_addr) + n); +#endif + + return ret; +} + +static inline void tcp_addr_to_sockaddr_in(void *dest, + const union tcp_addr *src, + unsigned int port) +{ + sockaddr_af *out = dest; + + memset(out, 0, sizeof(*out)); +#ifdef IPV6_TEST + out->sin6_family = AF_INET6; + out->sin6_port = port; + out->sin6_addr = src->a6; +#else + out->sin_family = AF_INET; + out->sin_port = port; + out->sin_addr = src->a4; +#endif +} + +static inline void test_init(unsigned int ntests, + thread_fn peer1, thread_fn peer2) +{ + test_init2(ntests, peer1, peer2, TEST_FAMILY, TEST_PREFIX, + TEST_SERVER_IP, TEST_CLIENT_IP); +} +extern void synchronize_threads(void); +extern void switch_ns(int fd); + +extern __thread union tcp_addr this_ip_addr; +extern __thread union tcp_addr this_ip_dest; +extern int test_family; + +extern void randomize_buffer(void *buf, size_t buflen); +extern int open_netns(void); +extern int unshare_open_netns(void); +extern const char veth_name[]; +extern int add_veth(const char *name, int nsfda, int nsfdb); +extern int add_vrf(const char *name, uint32_t tabid, int ifindex, int nsfd); +extern int ip_addr_add(const char *intf, int family, + union tcp_addr addr, uint8_t prefix); +extern int ip_route_add(const char *intf, int family, + union tcp_addr src, union tcp_addr dst); +extern int ip_route_add_vrf(const char *intf, int family, + union tcp_addr src, union tcp_addr dst, + uint8_t vrf); +extern int link_set_up(const char *intf); + +extern const unsigned int test_server_port; +extern int test_wait_fd(int sk, time_t sec, bool write); +extern int __test_connect_socket(int sk, const char *device, + void *addr, size_t addr_sz, time_t timeout); +extern int __test_listen_socket(int backlog, void *addr, size_t addr_sz); + +static inline int test_listen_socket(const union tcp_addr taddr, + unsigned int port, int backlog) +{ + sockaddr_af addr; + + tcp_addr_to_sockaddr_in(&addr, &taddr, htons(port)); + return __test_listen_socket(backlog, (void *)&addr, sizeof(addr)); +} + +/* + * In order for selftests to work under CONFIG_CRYPTO_FIPS=y, + * the password should be loger than 14 bytes, see hmac_setkey() + */ +#define TEST_TCP_AO_MINKEYLEN 14 +#define DEFAULT_TEST_PASSWORD "In this hour, I do not believe that any darkness will endure." + +#ifndef DEFAULT_TEST_ALGO +#define DEFAULT_TEST_ALGO "cmac(aes128)" +#endif + +#ifdef IPV6_TEST +#define DEFAULT_TEST_PREFIX 128 +#else +#define DEFAULT_TEST_PREFIX 32 +#endif + +/* + * Timeout on syscalls where failure is not expected. + * You may want to rise it if the test machine is very busy. + */ +#ifndef TEST_TIMEOUT_SEC +#define TEST_TIMEOUT_SEC 5 +#endif + +/* + * Timeout on connect() where a failure is expected. + * If set to 0 - kernel will try to retransmit SYN number of times, set in + * /proc/sys/net/ipv4/tcp_syn_retries + * By default set to 1 to make tests pass faster on non-busy machine. + */ +#ifndef TEST_RETRANSMIT_SEC +#define TEST_RETRANSMIT_SEC 1 +#endif + +static inline int _test_connect_socket(int sk, const union tcp_addr taddr, + unsigned int port, time_t timeout) +{ + sockaddr_af addr; + + tcp_addr_to_sockaddr_in(&addr, &taddr, htons(port)); + return __test_connect_socket(sk, veth_name, + (void *)&addr, sizeof(addr), timeout); +} + +static inline int test_connect_socket(int sk, const union tcp_addr taddr, + unsigned int port) +{ + return _test_connect_socket(sk, taddr, port, TEST_TIMEOUT_SEC); +} + +extern int __test_set_md5(int sk, void *addr, size_t addr_sz, + uint8_t prefix, int vrf, const char *password); +static inline int test_set_md5(int sk, const union tcp_addr in_addr, + uint8_t prefix, int vrf, const char *password) +{ + sockaddr_af addr; + + if (prefix > DEFAULT_TEST_PREFIX) + prefix = DEFAULT_TEST_PREFIX; + + tcp_addr_to_sockaddr_in(&addr, &in_addr, 0); + return __test_set_md5(sk, (void *)&addr, sizeof(addr), + prefix, vrf, password); +} + +extern int test_prepare_key_sockaddr(struct tcp_ao_add *ao, const char *alg, + void *addr, size_t addr_sz, bool set_current, bool set_rnext, + uint8_t prefix, uint8_t vrf, + uint8_t sndid, uint8_t rcvid, uint8_t maclen, + uint8_t keyflags, uint8_t keylen, const char *key); + +static inline int test_prepare_key(struct tcp_ao_add *ao, + const char *alg, union tcp_addr taddr, + bool set_current, bool set_rnext, + uint8_t prefix, uint8_t vrf, + uint8_t sndid, uint8_t rcvid, uint8_t maclen, + uint8_t keyflags, uint8_t keylen, const char *key) +{ + sockaddr_af addr; + + tcp_addr_to_sockaddr_in(&addr, &taddr, 0); + return test_prepare_key_sockaddr(ao, alg, (void *)&addr, sizeof(addr), + set_current, set_rnext, prefix, vrf, sndid, rcvid, + maclen, keyflags, keylen, key); +} + +static inline int test_prepare_def_key(struct tcp_ao_add *ao, + const char *key, uint8_t keyflags, + union tcp_addr in_addr, uint8_t prefix, uint8_t vrf, + uint8_t sndid, uint8_t rcvid) +{ + if (prefix > DEFAULT_TEST_PREFIX) + prefix = DEFAULT_TEST_PREFIX; + + return test_prepare_key(ao, DEFAULT_TEST_ALGO, in_addr, false, false, + prefix, vrf, sndid, rcvid, 0, keyflags, + strlen(key), key); +} + +extern int test_get_one_ao(int sk, struct tcp_ao_getsockopt *out, + void *addr, size_t addr_sz, + uint8_t prefix, uint8_t sndid, uint8_t rcvid); +extern int test_get_ao_info(int sk, struct tcp_ao_info_opt *out); +extern int test_set_ao_info(int sk, struct tcp_ao_info_opt *in); +extern int test_cmp_getsockopt_setsockopt(const struct tcp_ao_add *a, + const struct tcp_ao_getsockopt *b); +extern int test_cmp_getsockopt_setsockopt_ao(const struct tcp_ao_info_opt *a, + const struct tcp_ao_info_opt *b); + +static inline int test_verify_socket_key(int sk, struct tcp_ao_add *key) +{ + struct tcp_ao_getsockopt key2 = {}; + int err; + + err = test_get_one_ao(sk, &key2, &key->addr, sizeof(key->addr), + key->prefix, key->sndid, key->rcvid); + if (err) + return err; + + return test_cmp_getsockopt_setsockopt(key, &key2); +} + +static inline int test_add_key_vrf(int sk, + const char *key, uint8_t keyflags, + union tcp_addr in_addr, uint8_t prefix, + uint8_t vrf, uint8_t sndid, uint8_t rcvid) +{ + struct tcp_ao_add tmp = {}; + int err; + + err = test_prepare_def_key(&tmp, key, keyflags, in_addr, prefix, + vrf, sndid, rcvid); + if (err) + return err; + + err = setsockopt(sk, IPPROTO_TCP, TCP_AO_ADD_KEY, &tmp, sizeof(tmp)); + if (err < 0) + return -errno; + + return test_verify_socket_key(sk, &tmp); +} + +static inline int test_add_key(int sk, const char *key, + union tcp_addr in_addr, uint8_t prefix, + uint8_t sndid, uint8_t rcvid) +{ + return test_add_key_vrf(sk, key, 0, in_addr, prefix, 0, sndid, rcvid); +} + +static inline int test_verify_socket_ao(int sk, struct tcp_ao_info_opt *ao) +{ + struct tcp_ao_info_opt ao2 = {}; + int err; + + err = test_get_ao_info(sk, &ao2); + if (err) + return err; + + return test_cmp_getsockopt_setsockopt_ao(ao, &ao2); +} + +static inline int test_set_ao_flags(int sk, bool ao_required, bool accept_icmps) +{ + struct tcp_ao_info_opt ao = {}; + int err; + + err = test_get_ao_info(sk, &ao); + /* Maybe ao_info wasn't allocated yet */ + if (err && err != -ENOENT) + return err; + + ao.ao_required = !!ao_required; + ao.accept_icmps = !!accept_icmps; + err = test_set_ao_info(sk, &ao); + if (err) + return err; + + return test_verify_socket_ao(sk, &ao); +} + +extern ssize_t test_server_run(int sk, ssize_t quota, time_t timeout_sec); +extern ssize_t test_client_loop(int sk, char *buf, size_t buf_sz, + const size_t msg_len, time_t timeout_sec); +extern int test_client_verify(int sk, const size_t msg_len, const size_t nr, + time_t timeout_sec); + +struct tcp_ao_key_counters { + uint8_t sndid; + uint8_t rcvid; + uint64_t pkt_good; + uint64_t pkt_bad; +}; + +struct tcp_ao_counters { + /* per-netns */ + uint64_t netns_ao_good; + uint64_t netns_ao_bad; + uint64_t netns_ao_key_not_found; + uint64_t netns_ao_required; + uint64_t netns_ao_dropped_icmp; + /* per-socket */ + uint64_t ao_info_pkt_good; + uint64_t ao_info_pkt_bad; + uint64_t ao_info_pkt_key_not_found; + uint64_t ao_info_pkt_ao_required; + uint64_t ao_info_pkt_dropped_icmp; + /* per-key */ + size_t nr_keys; + struct tcp_ao_key_counters *key_cnts; +}; +extern int test_get_tcp_ao_counters(int sk, struct tcp_ao_counters *out); + +#define TEST_CNT_KEY_GOOD BIT(0) +#define TEST_CNT_KEY_BAD BIT(1) +#define TEST_CNT_SOCK_GOOD BIT(2) +#define TEST_CNT_SOCK_BAD BIT(3) +#define TEST_CNT_SOCK_KEY_NOT_FOUND BIT(4) +#define TEST_CNT_SOCK_AO_REQUIRED BIT(5) +#define TEST_CNT_SOCK_DROPPED_ICMP BIT(6) +#define TEST_CNT_NS_GOOD BIT(7) +#define TEST_CNT_NS_BAD BIT(8) +#define TEST_CNT_NS_KEY_NOT_FOUND BIT(9) +#define TEST_CNT_NS_AO_REQUIRED BIT(10) +#define TEST_CNT_NS_DROPPED_ICMP BIT(11) +typedef uint16_t test_cnt; + +#define TEST_CNT_AO_GOOD (TEST_CNT_SOCK_GOOD | TEST_CNT_NS_GOOD) +#define TEST_CNT_AO_BAD (TEST_CNT_SOCK_BAD | TEST_CNT_NS_BAD) +#define TEST_CNT_AO_KEY_NOT_FOUND (TEST_CNT_SOCK_KEY_NOT_FOUND | \ + TEST_CNT_NS_KEY_NOT_FOUND) +#define TEST_CNT_AO_REQUIRED (TEST_CNT_SOCK_AO_REQUIRED | \ + TEST_CNT_NS_AO_REQUIRED) +#define TEST_CNT_AO_DROPPED_ICMP (TEST_CNT_SOCK_DROPPED_ICMP | \ + TEST_CNT_NS_DROPPED_ICMP) +#define TEST_CNT_GOOD (TEST_CNT_KEY_GOOD | TEST_CNT_AO_GOOD) +#define TEST_CNT_BAD (TEST_CNT_KEY_BAD | TEST_CNT_AO_BAD) + +extern int __test_tcp_ao_counters_cmp(const char *tst_name, + struct tcp_ao_counters *before, struct tcp_ao_counters *after, + test_cnt expected); +extern int test_tcp_ao_key_counters_cmp(const char *tst_name, + struct tcp_ao_counters *before, struct tcp_ao_counters *after, + test_cnt expected, int sndid, int rcvid); +extern void test_tcp_ao_counters_free(struct tcp_ao_counters *cnts); +/* + * Frees buffers allocated in test_get_tcp_ao_counters(). + * The function doesn't expect new keys or keys removed between calls + * to test_get_tcp_ao_counters(). Check key counters manually if they + * may change. + */ +static inline int test_tcp_ao_counters_cmp(const char *tst_name, + struct tcp_ao_counters *before, + struct tcp_ao_counters *after, + test_cnt expected) +{ + int ret; + + ret = __test_tcp_ao_counters_cmp(tst_name, before, after, expected); + if (ret) + goto out; + ret = test_tcp_ao_key_counters_cmp(tst_name, before, after, + expected, -1, -1); +out: + test_tcp_ao_counters_free(before); + test_tcp_ao_counters_free(after); + return ret; +} + +struct netstat; +extern struct netstat *netstat_read(void); +extern void netstat_free(struct netstat *ns); +extern void netstat_print_diff(struct netstat *nsa, struct netstat *nsb); +extern uint64_t netstat_get(struct netstat *ns, + const char *name, bool *not_found); + +static inline uint64_t netstat_get_one(const char *name, bool *not_found) +{ + struct netstat *ns = netstat_read(); + uint64_t ret; + + ret = netstat_get(ns, name, not_found); + + netstat_free(ns); + return ret; +} + +struct tcp_sock_queue { + uint32_t seq; + void *buf; +}; + +struct tcp_sock_state { + struct tcp_info info; + struct tcp_repair_window trw; + struct tcp_sock_queue out; + int outq_len; /* output queue size (not sent + not acked) */ + int outq_nsd_len; /* output queue size (not sent only) */ + struct tcp_sock_queue in; + int inq_len; + int mss; + int timestamp; +}; + +extern void __test_sock_checkpoint(int sk, struct tcp_sock_state *state, + void *addr, size_t addr_size); +static inline void test_sock_checkpoint(int sk, struct tcp_sock_state *state, + sockaddr_af *saddr) +{ + __test_sock_checkpoint(sk, state, saddr, sizeof(*saddr)); +} +extern void test_ao_checkpoint(int sk, struct tcp_ao_repair *state); +extern void __test_sock_restore(int sk, const char *device, + struct tcp_sock_state *state, + void *saddr, void *daddr, size_t addr_size); +static inline void test_sock_restore(int sk, struct tcp_sock_state *state, + sockaddr_af *saddr, + const union tcp_addr daddr, + unsigned int dport) +{ + sockaddr_af addr; + + tcp_addr_to_sockaddr_in(&addr, &daddr, htons(dport)); + __test_sock_restore(sk, veth_name, state, saddr, &addr, sizeof(addr)); +} +extern void test_ao_restore(int sk, struct tcp_ao_repair *state); +extern void test_sock_state_free(struct tcp_sock_state *state); +extern void test_enable_repair(int sk); +extern void test_disable_repair(int sk); +extern void test_kill_sk(int sk); +static inline int test_add_repaired_key(int sk, + const char *key, uint8_t keyflags, + union tcp_addr in_addr, uint8_t prefix, + uint8_t sndid, uint8_t rcvid) +{ + struct tcp_ao_add tmp = {}; + int err; + + err = test_prepare_def_key(&tmp, key, keyflags, in_addr, prefix, + 0, sndid, rcvid); + if (err) + return err; + + tmp.set_current = 1; + tmp.set_rnext = 1; + if (setsockopt(sk, IPPROTO_TCP, TCP_AO_ADD_KEY, &tmp, sizeof(tmp)) < 0) + return -errno; + + return test_verify_socket_key(sk, &tmp); +} + +#endif /* _AOLIB_H_ */ diff --git a/tools/testing/selftests/net/tcp_ao/lib/kconfig.c b/tools/testing/selftests/net/tcp_ao/lib/kconfig.c new file mode 100644 index 00000000000000..f279ffc3843b56 --- /dev/null +++ b/tools/testing/selftests/net/tcp_ao/lib/kconfig.c @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Check what features does the kernel support (where the selftest is running). + * Somewhat inspired by CRIU kerndat/kdat kernel features detector. + */ +#include +#include "aolib.h" + +struct kconfig_t { + int _errno; /* the returned error if not supported */ + int (*check_kconfig)(int *error); +}; + +static int has_net_ns(int *err) +{ + if (access("/proc/self/ns/net", F_OK) < 0) { + *err = errno; + if (errno == ENOENT) + return 0; + test_print("Unable to access /proc/self/ns/net: %m"); + return -errno; + } + return *err = errno = 0; +} + +static int has_veth(int *err) +{ + int orig_netns, ns_a, ns_b; + + orig_netns = open_netns(); + ns_a = unshare_open_netns(); + ns_b = unshare_open_netns(); + + *err = add_veth("check_veth", ns_a, ns_b); + + switch_ns(orig_netns); + close(orig_netns); + close(ns_a); + close(ns_b); + return 0; +} + +static int has_tcp_ao(int *err) +{ + struct sockaddr_in addr = { + .sin_family = test_family, + }; + struct tcp_ao_add tmp = {}; + const char *password = DEFAULT_TEST_PASSWORD; + int sk, ret = 0; + + sk = socket(test_family, SOCK_STREAM, IPPROTO_TCP); + if (sk < 0) { + test_print("socket(): %m"); + return -errno; + } + + tmp.sndid = 100; + tmp.rcvid = 100; + tmp.keylen = strlen(password); + memcpy(tmp.key, password, strlen(password)); + strcpy(tmp.alg_name, "hmac(sha1)"); + memcpy(&tmp.addr, &addr, sizeof(addr)); + *err = 0; + if (setsockopt(sk, IPPROTO_TCP, TCP_AO_ADD_KEY, &tmp, sizeof(tmp)) < 0) { + *err = errno; + if (errno != ENOPROTOOPT) + ret = -errno; + } + close(sk); + return ret; +} + +static int has_tcp_md5(int *err) +{ + union tcp_addr addr_any = {}; + int sk, ret = 0; + + sk = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + if (sk < 0) { + test_print("socket(): %m"); + return -errno; + } + + /* + * Under CONFIG_CRYPTO_FIPS=y it fails with ENOMEM, rather with + * anything more descriptive. Oh well. + */ + *err = 0; + if (test_set_md5(sk, addr_any, 0, -1, DEFAULT_TEST_PASSWORD)) { + *err = errno; + if (errno != ENOPROTOOPT && errno == ENOMEM) { + test_print("setsockopt(TCP_MD5SIG_EXT): %m"); + ret = -errno; + } + } + close(sk); + return ret; +} + +static int has_vrfs(int *err) +{ + int orig_netns, ns_test, ret = 0; + + orig_netns = open_netns(); + ns_test = unshare_open_netns(); + + *err = add_vrf("ksft-check", 55, 101, ns_test); + if (*err && *err != -EOPNOTSUPP) { + test_print("Failed to add a VRF: %d", *err); + ret = *err; + } + + switch_ns(orig_netns); + close(orig_netns); + close(ns_test); + return ret; +} + +static pthread_mutex_t kconfig_lock = PTHREAD_MUTEX_INITIALIZER; +static struct kconfig_t kconfig[__KCONFIG_LAST__] = { + { -1, has_net_ns }, + { -1, has_veth }, + { -1, has_tcp_ao }, + { -1, has_tcp_md5 }, + { -1, has_vrfs }, +}; + +const char *tests_skip_reason[__KCONFIG_LAST__] = { + "Tests require network namespaces support (CONFIG_NET_NS)", + "Tests require veth support (CONFIG_VETH)", + "Tests require TCP-AO support (CONFIG_TCP_AO)", + "setsockopt(TCP_MD5SIG_EXT) is not supported (CONFIG_TCP_MD5)", + "VRFs are not supported (CONFIG_NET_VRF)", +}; + +bool kernel_config_has(enum test_needs_kconfig k) +{ + bool ret; + + pthread_mutex_lock(&kconfig_lock); + if (kconfig[k]._errno == -1) { + if (kconfig[k].check_kconfig(&kconfig[k]._errno)) + test_error("Failed to initialize kconfig %u", k); + } + ret = kconfig[k]._errno == 0; + pthread_mutex_unlock(&kconfig_lock); + return ret; +} diff --git a/tools/testing/selftests/net/tcp_ao/lib/netlink.c b/tools/testing/selftests/net/tcp_ao/lib/netlink.c new file mode 100644 index 00000000000000..b731f2c8408318 --- /dev/null +++ b/tools/testing/selftests/net/tcp_ao/lib/netlink.c @@ -0,0 +1,415 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Original from tools/testing/selftests/net/ipsec.c */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "aolib.h" + +#define MAX_PAYLOAD 2048 + +static int netlink_sock(int *sock, uint32_t *seq_nr, int proto) +{ + if (*sock > 0) { + seq_nr++; + return 0; + } + + *sock = socket(AF_NETLINK, SOCK_RAW | SOCK_CLOEXEC, proto); + if (*sock < 0) { + test_print("socket(AF_NETLINK)"); + return -1; + } + + randomize_buffer(seq_nr, sizeof(*seq_nr)); + + return 0; +} + +static int netlink_check_answer(int sock, bool quite) +{ + struct nlmsgerror { + struct nlmsghdr hdr; + int error; + struct nlmsghdr orig_msg; + } answer; + + if (recv(sock, &answer, sizeof(answer), 0) < 0) { + test_print("recv()"); + return -1; + } else if (answer.hdr.nlmsg_type != NLMSG_ERROR) { + test_print("expected NLMSG_ERROR, got %d", + (int)answer.hdr.nlmsg_type); + return -1; + } else if (answer.error) { + if (!quite) { + test_print("NLMSG_ERROR: %d: %s", + answer.error, strerror(-answer.error)); + } + return answer.error; + } + + return 0; +} + +static inline struct rtattr *rtattr_hdr(struct nlmsghdr *nh) +{ + return (struct rtattr *)((char *)(nh) + RTA_ALIGN((nh)->nlmsg_len)); +} + +static int rtattr_pack(struct nlmsghdr *nh, size_t req_sz, + unsigned short rta_type, const void *payload, size_t size) +{ + /* NLMSG_ALIGNTO == RTA_ALIGNTO, nlmsg_len already aligned */ + struct rtattr *attr = rtattr_hdr(nh); + size_t nl_size = RTA_ALIGN(nh->nlmsg_len) + RTA_LENGTH(size); + + if (req_sz < nl_size) { + test_print("req buf is too small: %zu < %zu", req_sz, nl_size); + return -1; + } + nh->nlmsg_len = nl_size; + + attr->rta_len = RTA_LENGTH(size); + attr->rta_type = rta_type; + memcpy(RTA_DATA(attr), payload, size); + + return 0; +} + +static struct rtattr *_rtattr_begin(struct nlmsghdr *nh, size_t req_sz, + unsigned short rta_type, const void *payload, size_t size) +{ + struct rtattr *ret = rtattr_hdr(nh); + + if (rtattr_pack(nh, req_sz, rta_type, payload, size)) + return 0; + + return ret; +} + +static inline struct rtattr *rtattr_begin(struct nlmsghdr *nh, size_t req_sz, + unsigned short rta_type) +{ + return _rtattr_begin(nh, req_sz, rta_type, 0, 0); +} + +static inline void rtattr_end(struct nlmsghdr *nh, struct rtattr *attr) +{ + char *nlmsg_end = (char *)nh + nh->nlmsg_len; + + attr->rta_len = nlmsg_end - (char *)attr; +} + +static int veth_pack_peerb(struct nlmsghdr *nh, size_t req_sz, + const char *peer, int ns) +{ + struct ifinfomsg pi; + struct rtattr *peer_attr; + + memset(&pi, 0, sizeof(pi)); + pi.ifi_family = AF_UNSPEC; + pi.ifi_change = 0xFFFFFFFF; + + peer_attr = _rtattr_begin(nh, req_sz, VETH_INFO_PEER, &pi, sizeof(pi)); + if (!peer_attr) + return -1; + + if (rtattr_pack(nh, req_sz, IFLA_IFNAME, peer, strlen(peer))) + return -1; + + if (rtattr_pack(nh, req_sz, IFLA_NET_NS_FD, &ns, sizeof(ns))) + return -1; + + rtattr_end(nh, peer_attr); + + return 0; +} + +static int __add_veth(int sock, uint32_t seq, const char *name, + int ns_a, int ns_b) +{ + uint16_t flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE; + struct { + struct nlmsghdr nh; + struct ifinfomsg info; + char attrbuf[MAX_PAYLOAD]; + } req; + static const char veth_type[] = "veth"; + struct rtattr *link_info, *info_data; + + memset(&req, 0, sizeof(req)); + req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(req.info)); + req.nh.nlmsg_type = RTM_NEWLINK; + req.nh.nlmsg_flags = flags; + req.nh.nlmsg_seq = seq; + req.info.ifi_family = AF_UNSPEC; + req.info.ifi_change = 0xFFFFFFFF; + + if (rtattr_pack(&req.nh, sizeof(req), IFLA_IFNAME, name, strlen(name))) + return -1; + + if (rtattr_pack(&req.nh, sizeof(req), IFLA_NET_NS_FD, &ns_a, sizeof(ns_a))) + return -1; + + link_info = rtattr_begin(&req.nh, sizeof(req), IFLA_LINKINFO); + if (!link_info) + return -1; + + if (rtattr_pack(&req.nh, sizeof(req), IFLA_INFO_KIND, veth_type, sizeof(veth_type))) + return -1; + + info_data = rtattr_begin(&req.nh, sizeof(req), IFLA_INFO_DATA); + if (!info_data) + return -1; + + if (veth_pack_peerb(&req.nh, sizeof(req), name, ns_b)) + return -1; + + rtattr_end(&req.nh, info_data); + rtattr_end(&req.nh, link_info); + + if (send(sock, &req, req.nh.nlmsg_len, 0) < 0) { + test_print("send()"); + return -1; + } + return netlink_check_answer(sock, false); +} + +int add_veth(const char *name, int nsfda, int nsfdb) +{ + int route_sock = -1, ret; + uint32_t route_seq; + + if (netlink_sock(&route_sock, &route_seq, NETLINK_ROUTE)) + test_error("Failed to open netlink route socket\n"); + + ret = __add_veth(route_sock, route_seq++, name, nsfda, nsfdb); + close(route_sock); + return ret; +} + +static int __ip_addr_add(int sock, uint32_t seq, const char *intf, + int family, union tcp_addr addr, uint8_t prefix) +{ + uint16_t flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE; + struct { + struct nlmsghdr nh; + struct ifaddrmsg info; + char attrbuf[MAX_PAYLOAD]; + } req; + size_t addr_len = (family == AF_INET) ? sizeof(struct in_addr) : + sizeof(struct in6_addr); + + memset(&req, 0, sizeof(req)); + req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(req.info)); + req.nh.nlmsg_type = RTM_NEWADDR; + req.nh.nlmsg_flags = flags; + req.nh.nlmsg_seq = seq; + req.info.ifa_family = family; + req.info.ifa_prefixlen = prefix; + req.info.ifa_index = if_nametoindex(intf); + req.info.ifa_flags = IFA_F_NODAD; + + if (rtattr_pack(&req.nh, sizeof(req), IFA_LOCAL, &addr, addr_len)) + return -1; + + if (send(sock, &req, req.nh.nlmsg_len, 0) < 0) { + test_print("send()"); + return -1; + } + return netlink_check_answer(sock, true); +} + +int ip_addr_add(const char *intf, int family, + union tcp_addr addr, uint8_t prefix) +{ + int route_sock = -1, ret; + uint32_t route_seq; + + if (netlink_sock(&route_sock, &route_seq, NETLINK_ROUTE)) + test_error("Failed to open netlink route socket\n"); + + ret = __ip_addr_add(route_sock, route_seq++, intf, + family, addr, prefix); + + close(route_sock); + return ret; +} + +static int __ip_route_add(int sock, uint32_t seq, const char *intf, int family, + union tcp_addr src, union tcp_addr dst, uint8_t vrf) +{ + struct { + struct nlmsghdr nh; + struct rtmsg rt; + char attrbuf[MAX_PAYLOAD]; + } req; + unsigned int index = if_nametoindex(intf); + size_t addr_len = (family == AF_INET) ? sizeof(struct in_addr) : + sizeof(struct in6_addr); + + memset(&req, 0, sizeof(req)); + req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(req.rt)); + req.nh.nlmsg_type = RTM_NEWROUTE; + req.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_CREATE; + req.nh.nlmsg_seq = seq; + req.rt.rtm_family = family; + req.rt.rtm_dst_len = (family == AF_INET) ? 32 : 128; + req.rt.rtm_table = RT_TABLE_MAIN; + req.rt.rtm_protocol = RTPROT_BOOT; + req.rt.rtm_scope = RT_SCOPE_UNIVERSE; + req.rt.rtm_type = RTN_UNICAST; + + if (rtattr_pack(&req.nh, sizeof(req), RTA_DST, &dst, addr_len)) + return -1; + + if (rtattr_pack(&req.nh, sizeof(req), RTA_PREFSRC, &src, addr_len)) + return -1; + + if (rtattr_pack(&req.nh, sizeof(req), RTA_OIF, &index, sizeof(index))) + return -1; + + if (send(sock, &req, req.nh.nlmsg_len, 0) < 0) { + test_print("send()"); + return -1; + } + + return netlink_check_answer(sock, true); +} + +int ip_route_add_vrf(const char *intf, int family, + union tcp_addr src, union tcp_addr dst, uint8_t vrf) +{ + int route_sock = -1, ret; + uint32_t route_seq; + + if (netlink_sock(&route_sock, &route_seq, NETLINK_ROUTE)) + test_error("Failed to open netlink route socket\n"); + + ret = __ip_route_add(route_sock, route_seq++, intf, + family, src, dst, vrf); + if (ret == -EEXIST) /* ignoring */ + ret = 0; + + close(route_sock); + return ret; +} + +int ip_route_add(const char *intf, int family, + union tcp_addr src, union tcp_addr dst) +{ + return ip_route_add_vrf(intf, family, src, dst, RT_TABLE_MAIN); +} + +static int __link_set_up(int sock, uint32_t seq, const char *intf) +{ + struct { + struct nlmsghdr nh; + struct ifinfomsg info; + char attrbuf[MAX_PAYLOAD]; + } req; + + memset(&req, 0, sizeof(req)); + req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(req.info)); + req.nh.nlmsg_type = RTM_NEWLINK; + req.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK; + req.nh.nlmsg_seq = seq; + req.info.ifi_family = AF_UNSPEC; + req.info.ifi_change = 0xFFFFFFFF; + req.info.ifi_index = if_nametoindex(intf); + req.info.ifi_flags = IFF_UP; + req.info.ifi_change = IFF_UP; + + if (send(sock, &req, req.nh.nlmsg_len, 0) < 0) { + test_print("send()"); + return -1; + } + return netlink_check_answer(sock, false); +} + +int link_set_up(const char *intf) +{ + int route_sock = -1, ret; + uint32_t route_seq; + + if (netlink_sock(&route_sock, &route_seq, NETLINK_ROUTE)) + test_error("Failed to open netlink route socket\n"); + + ret = __link_set_up(route_sock, route_seq++, intf); + + close(route_sock); + return ret; +} + +static int __add_vrf(int sock, uint32_t seq, const char *name, + uint32_t tabid, int ifindex, int nsfd) +{ + uint16_t flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE; + struct { + struct nlmsghdr nh; + struct ifinfomsg info; + char attrbuf[MAX_PAYLOAD]; + } req; + static const char vrf_type[] = "vrf"; + struct rtattr *link_info, *info_data; + + memset(&req, 0, sizeof(req)); + req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(req.info)); + req.nh.nlmsg_type = RTM_NEWLINK; + req.nh.nlmsg_flags = flags; + req.nh.nlmsg_seq = seq; + req.info.ifi_family = AF_UNSPEC; + req.info.ifi_change = 0xFFFFFFFF; + req.info.ifi_index = ifindex; + + if (rtattr_pack(&req.nh, sizeof(req), IFLA_IFNAME, name, strlen(name))) + return -1; + + if (nsfd >= 0) + if (rtattr_pack(&req.nh, sizeof(req), IFLA_NET_NS_FD, + &nsfd, sizeof(nsfd))) + return -1; + + link_info = rtattr_begin(&req.nh, sizeof(req), IFLA_LINKINFO); + if (!link_info) + return -1; + + if (rtattr_pack(&req.nh, sizeof(req), IFLA_INFO_KIND, vrf_type, sizeof(vrf_type))) + return -1; + + info_data = rtattr_begin(&req.nh, sizeof(req), IFLA_INFO_DATA); + if (!info_data) + return -1; + + if (rtattr_pack(&req.nh, sizeof(req), IFLA_VRF_TABLE, + &tabid, sizeof(tabid))) + return -1; + + rtattr_end(&req.nh, info_data); + rtattr_end(&req.nh, link_info); + + if (send(sock, &req, req.nh.nlmsg_len, 0) < 0) { + test_print("send()"); + return -1; + } + return netlink_check_answer(sock, true); +} + +int add_vrf(const char *name, uint32_t tabid, int ifindex, int nsfd) +{ + int route_sock = -1, ret; + uint32_t route_seq; + + if (netlink_sock(&route_sock, &route_seq, NETLINK_ROUTE)) + test_error("Failed to open netlink route socket\n"); + + ret = __add_vrf(route_sock, route_seq++, name, tabid, ifindex, nsfd); + close(route_sock); + return ret; +} diff --git a/tools/testing/selftests/net/tcp_ao/lib/proc.c b/tools/testing/selftests/net/tcp_ao/lib/proc.c new file mode 100644 index 00000000000000..2322f4d4676d0c --- /dev/null +++ b/tools/testing/selftests/net/tcp_ao/lib/proc.c @@ -0,0 +1,273 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include "../../../../../include/linux/compiler.h" +#include "../../../../../include/linux/kernel.h" +#include "aolib.h" + +struct netstat_counter { + uint64_t val; + char *name; +}; + +struct netstat { + char *header_name; + struct netstat *next; + size_t counters_nr; + struct netstat_counter *counters; +}; + +static struct netstat *lookup_type(struct netstat *ns, + const char *type, size_t len) +{ + while (ns != NULL) { + size_t cmp = max(len, strlen(ns->header_name)); + + if (!strncmp(ns->header_name, type, cmp)) + return ns; + ns = ns->next; + } + return NULL; +} + +static struct netstat *lookup_get(struct netstat *ns, + const char *type, const size_t len) +{ + struct netstat *ret; + + ret = lookup_type(ns, type, len); + if (ret != NULL) + return ret; + + ret = malloc(sizeof(struct netstat)); + if (!ret) + test_error("malloc()"); + + ret->header_name = strndup(type, len); + if (ret->header_name == NULL) + test_error("strndup()"); + ret->next = ns; + ret->counters_nr = 0; + ret->counters = NULL; + + return ret; +} + +static struct netstat *lookup_get_column(struct netstat *ns, const char *line) +{ + char *column; + + column = strchr(line, ':'); + if (!column) + test_error("can't parse netstat file"); + + return lookup_get(ns, line, column - line); +} + +static void netstat_read_type(FILE *fnetstat, struct netstat **dest, char *line) +{ + struct netstat *type = lookup_get_column(*dest, line); + const char *pos = line; + size_t i, nr_elems = 0; + char tmp; + + while ((pos = strchr(pos, ' '))) { + nr_elems++; + pos++; + } + + *dest = type; + type->counters = reallocarray(type->counters, + type->counters_nr + nr_elems, + sizeof(struct netstat_counter)); + if (!type->counters) + test_error("reallocarray()"); + + pos = strchr(line, ' ') + 1; + + if (fscanf(fnetstat, type->header_name) == EOF) + test_error("fscanf(%s)", type->header_name); + if (fread(&tmp, 1, 1, fnetstat) != 1 || tmp != ':') + test_error("Unexpected netstat format (%c)", tmp); + + for (i = type->counters_nr; i < type->counters_nr + nr_elems; i++) { + struct netstat_counter *nc = &type->counters[i]; + const char *new_pos = strchr(pos, ' '); + const char *fmt = " %" PRIu64; + + if (new_pos == NULL) + new_pos = strchr(pos, '\n'); + + nc->name = strndup(pos, new_pos - pos); + if (nc->name == NULL) + test_error("strndup()"); + + if (unlikely(!strcmp(nc->name, "MaxConn"))) + fmt = " %" PRId64; /* MaxConn is signed, RFC 2012 */ + if (fscanf(fnetstat, fmt, &nc->val) != 1) + test_error("fscanf(%s)", nc->name); + pos = new_pos + 1; + } + type->counters_nr += nr_elems; + + if (fread(&tmp, 1, 1, fnetstat) != 1 || tmp != '\n') + test_error("Unexpected netstat format"); +} + +static const char *snmp6_name = "Snmp6"; +static void snmp6_read(FILE *fnetstat, struct netstat **dest) +{ + struct netstat *type = lookup_get(*dest, snmp6_name, strlen(snmp6_name)); + char *counter_name; + size_t i; + + for (i = type->counters_nr;; i++) { + struct netstat_counter *nc; + uint64_t counter; + + if (fscanf(fnetstat, "%ms", &counter_name) == EOF) + break; + if (fscanf(fnetstat, "%" PRIu64, &counter) == EOF) + test_error("Unexpected snmp6 format"); + type->counters = reallocarray(type->counters, i + 1, + sizeof(struct netstat_counter)); + if (!type->counters) + test_error("reallocarray()"); + nc = &type->counters[i]; + nc->name = counter_name; + nc->val = counter; + } + type->counters_nr = i; + *dest = type; +} + +struct netstat *netstat_read(void) +{ + struct netstat *ret = 0; + size_t line_sz = 0; + char *line = NULL; + FILE *fnetstat; + + /* + * Opening thread-self instead of /proc/net/... as the latter + * points to /proc/self/net/ which instantiates thread-leader's + * net-ns, see: + * commit 155134fef2b6 ("Revert "proc: Point /proc/{mounts,net} at..") + */ + errno = 0; + fnetstat = fopen("/proc/thread-self/net/netstat", "r"); + if (fnetstat == NULL) + test_error("failed to open /proc/net/netstat"); + + while (getline(&line, &line_sz, fnetstat) != -1) + netstat_read_type(fnetstat, &ret, line); + fclose(fnetstat); + + errno = 0; + fnetstat = fopen("/proc/thread-self/net/snmp", "r"); + if (fnetstat == NULL) + test_error("failed to open /proc/net/snmp"); + + while (getline(&line, &line_sz, fnetstat) != -1) + netstat_read_type(fnetstat, &ret, line); + fclose(fnetstat); + + errno = 0; + fnetstat = fopen("/proc/thread-self/net/snmp6", "r"); + if (fnetstat == NULL) + test_error("failed to open /proc/net/snmp6"); + + snmp6_read(fnetstat, &ret); + fclose(fnetstat); + + free(line); + return ret; +} + +void netstat_free(struct netstat *ns) +{ + while (ns != NULL) { + struct netstat *prev = ns; + size_t i; + + free(ns->header_name); + for (i = 0; i < ns->counters_nr; i++) + free(ns->counters[i].name); + free(ns->counters); + ns = ns->next; + free(prev); + } +} + +static inline void +__netstat_print_diff(uint64_t a, struct netstat *nsb, size_t i) +{ + if (unlikely(!strcmp(nsb->header_name, "MaxConn"))) { + test_print("%8s %25s: %" PRId64 " => %" PRId64, + nsb->header_name, nsb->counters[i].name, + a, nsb->counters[i].val); + return; + } + + test_print("%8s %25s: %" PRIu64 " => %" PRIu64, nsb->header_name, + nsb->counters[i].name, a, nsb->counters[i].val); +} + +void netstat_print_diff(struct netstat *nsa, struct netstat *nsb) +{ + size_t i, j; + + while (nsb != NULL) { + if (unlikely(strcmp(nsb->header_name, nsa->header_name))) { + for (i = 0; i < nsb->counters_nr; i++) + __netstat_print_diff(0, nsb, i); + nsb = nsb->next; + continue; + } + + if (nsb->counters_nr < nsa->counters_nr) + test_error("Unexpected: some counters dissapeared!"); + + for (j = 0, i = 0; i < nsb->counters_nr; i++) { + if (strcmp(nsb->counters[i].name, nsa->counters[j].name)) { + __netstat_print_diff(0, nsb, i); + continue; + } + + if (nsa->counters[j].val == nsb->counters[i].val) { + j++; + continue; + } + + __netstat_print_diff(nsa->counters[j].val, nsb, i); + j++; + } + if (j != nsa->counters_nr) + test_error("Unexpected: some counters dissapeared!"); + + nsb = nsb->next; + nsa = nsa->next; + } +} + +uint64_t netstat_get(struct netstat *ns, const char *name, bool *not_found) +{ + if (not_found) + *not_found = false; + + while (ns != NULL) { + size_t i; + + for (i = 0; i < ns->counters_nr; i++) { + if (!strcmp(name, ns->counters[i].name)) + return ns->counters[i].val; + } + + ns = ns->next; + } + + if (not_found) + *not_found = true; + return 0; +} diff --git a/tools/testing/selftests/net/tcp_ao/lib/repair.c b/tools/testing/selftests/net/tcp_ao/lib/repair.c new file mode 100644 index 00000000000000..9893b3ba69f5a9 --- /dev/null +++ b/tools/testing/selftests/net/tcp_ao/lib/repair.c @@ -0,0 +1,254 @@ +// SPDX-License-Identifier: GPL-2.0 +/* This is over-simplified TCP_REPAIR for TCP_ESTABLISHED sockets + * It tests that TCP-AO enabled connection can be restored. + * For the proper socket repair see: + * https://github.com/checkpoint-restore/criu/blob/criu-dev/soccr/soccr.h + */ +#include +#include +#include +#include "aolib.h" + +#ifndef TCPOPT_MAXSEG +# define TCPOPT_MAXSEG 2 +#endif +#ifndef TCPOPT_WINDOW +# define TCPOPT_WINDOW 3 +#endif +#ifndef TCPOPT_SACK_PERMITTED +# define TCPOPT_SACK_PERMITTED 4 +#endif +#ifndef TCPOPT_TIMESTAMP +# define TCPOPT_TIMESTAMP 8 +#endif + +enum { + TCP_ESTABLISHED = 1, + TCP_SYN_SENT, + TCP_SYN_RECV, + TCP_FIN_WAIT1, + TCP_FIN_WAIT2, + TCP_TIME_WAIT, + TCP_CLOSE, + TCP_CLOSE_WAIT, + TCP_LAST_ACK, + TCP_LISTEN, + TCP_CLOSING, /* Now a valid state */ + TCP_NEW_SYN_RECV, + + TCP_MAX_STATES /* Leave at the end! */ +}; + +static void test_sock_checkpoint_queue(int sk, int queue, int qlen, + struct tcp_sock_queue *q) +{ + socklen_t len; + int ret; + + if (setsockopt(sk, SOL_TCP, TCP_REPAIR_QUEUE, &queue, sizeof(queue))) + test_error("setsockopt(TCP_REPAIR_QUEUE)"); + + len = sizeof(q->seq); + ret = getsockopt(sk, SOL_TCP, TCP_QUEUE_SEQ, &q->seq, &len); + if (ret || len != sizeof(q->seq)) + test_error("getsockopt(TCP_QUEUE_SEQ): %d", (int)len); + + if (!qlen) { + q->buf = NULL; + return; + } + + q->buf = malloc(qlen); + if (q->buf == NULL) + test_error("malloc()"); + ret = recv(sk, q->buf, qlen, MSG_PEEK | MSG_DONTWAIT); + if (ret != qlen) + test_error("recv(%d): %d", qlen, ret); +} + +void __test_sock_checkpoint(int sk, struct tcp_sock_state *state, + void *addr, size_t addr_size) +{ + socklen_t len = sizeof(state->info); + int ret; + + memset(state, 0, sizeof(*state)); + + ret = getsockopt(sk, SOL_TCP, TCP_INFO, &state->info, &len); + if (ret || len != sizeof(state->info)) + test_error("getsockopt(TCP_INFO): %d", (int)len); + + len = addr_size; + if (getsockname(sk, addr, &len) || len != addr_size) + test_error("getsockname(): %d", (int)len); + + len = sizeof(state->trw); + ret = getsockopt(sk, SOL_TCP, TCP_REPAIR_WINDOW, &state->trw, &len); + if (ret || len != sizeof(state->trw)) + test_error("getsockopt(TCP_REPAIR_WINDOW): %d", (int)len); + + if (ioctl(sk, SIOCOUTQ, &state->outq_len)) + test_error("ioctl(SIOCOUTQ)"); + + if (ioctl(sk, SIOCOUTQNSD, &state->outq_nsd_len)) + test_error("ioctl(SIOCOUTQNSD)"); + test_sock_checkpoint_queue(sk, TCP_SEND_QUEUE, state->outq_len, &state->out); + + if (ioctl(sk, SIOCINQ, &state->inq_len)) + test_error("ioctl(SIOCINQ)"); + test_sock_checkpoint_queue(sk, TCP_RECV_QUEUE, state->inq_len, &state->in); + + if (state->info.tcpi_state == TCP_CLOSE) + state->outq_len = state->outq_nsd_len = 0; + + len = sizeof(state->mss); + ret = getsockopt(sk, SOL_TCP, TCP_MAXSEG, &state->mss, &len); + if (ret || len != sizeof(state->mss)) + test_error("getsockopt(TCP_MAXSEG): %d", (int)len); + + len = sizeof(state->timestamp); + ret = getsockopt(sk, SOL_TCP, TCP_TIMESTAMP, &state->timestamp, &len); + if (ret || len != sizeof(state->timestamp)) + test_error("getsockopt(TCP_TIMESTAMP): %d", (int)len); +} + +void test_ao_checkpoint(int sk, struct tcp_ao_repair *state) +{ + socklen_t len = sizeof(*state); + int ret; + + memset(state, 0, sizeof(*state)); + + ret = getsockopt(sk, SOL_TCP, TCP_AO_REPAIR, state, &len); + if (ret || len != sizeof(*state)) + test_error("getsockopt(TCP_AO_REPAIR): %d", (int)len); +} + +static void test_sock_restore_seq(int sk, int queue, uint32_t seq) +{ + if (setsockopt(sk, SOL_TCP, TCP_REPAIR_QUEUE, &queue, sizeof(queue))) + test_error("setsockopt(TCP_REPAIR_QUEUE)"); + + if (setsockopt(sk, SOL_TCP, TCP_QUEUE_SEQ, &seq, sizeof(seq))) + test_error("setsockopt(TCP_QUEUE_SEQ)"); +} + +static void test_sock_restore_queue(int sk, int queue, void *buf, int len) +{ + int chunk = len; + size_t off = 0; + + if (len == 0) + return; + + if (setsockopt(sk, SOL_TCP, TCP_REPAIR_QUEUE, &queue, sizeof(queue))) + test_error("setsockopt(TCP_REPAIR_QUEUE)"); + + do { + int ret; + + ret = send(sk, buf + off, chunk, 0); + if (ret <= 0) { + if (chunk > 1024) { + chunk >>= 1; + continue; + } + test_error("send()"); + } + off += ret; + len -= ret; + } while (len > 0); +} + +void __test_sock_restore(int sk, const char *device, + struct tcp_sock_state *state, + void *saddr, void *daddr, size_t addr_size) +{ + struct tcp_repair_opt opts[4]; + unsigned int opt_nr = 0; + long flags; + + if (bind(sk, saddr, addr_size)) + test_error("bind()"); + + flags = fcntl(sk, F_GETFL); + if ((flags < 0) || (fcntl(sk, F_SETFL, flags | O_NONBLOCK) < 0)) + test_error("fcntl()"); + + test_sock_restore_seq(sk, TCP_RECV_QUEUE, state->in.seq - state->inq_len); + test_sock_restore_seq(sk, TCP_SEND_QUEUE, state->out.seq - state->outq_len); + + if (device != NULL && setsockopt(sk, SOL_SOCKET, SO_BINDTODEVICE, + device, strlen(device) + 1)) + test_error("setsockopt(SO_BINDTODEVICE, %s)", device); + + if (connect(sk, daddr, addr_size)) + test_error("connect()"); + + if (state->info.tcpi_options & TCPI_OPT_SACK) { + opts[opt_nr].opt_code = TCPOPT_SACK_PERMITTED; + opts[opt_nr].opt_val = 0; + opt_nr++; + } + if (state->info.tcpi_options & TCPI_OPT_WSCALE) { + opts[opt_nr].opt_code = TCPOPT_WINDOW; + opts[opt_nr].opt_val = state->info.tcpi_snd_wscale + + (state->info.tcpi_rcv_wscale << 16); + opt_nr++; + } + if (state->info.tcpi_options & TCPI_OPT_TIMESTAMPS) { + opts[opt_nr].opt_code = TCPOPT_TIMESTAMP; + opts[opt_nr].opt_val = 0; + opt_nr++; + } + opts[opt_nr].opt_code = TCPOPT_MAXSEG; + opts[opt_nr].opt_val = state->mss; + opt_nr++; + + if (setsockopt(sk, SOL_TCP, TCP_REPAIR_OPTIONS, opts, opt_nr * sizeof(opts[0]))) + test_error("setsockopt(TCP_REPAIR_OPTIONS)"); + + if (state->info.tcpi_options & TCPI_OPT_TIMESTAMPS) { + if (setsockopt(sk, SOL_TCP, TCP_TIMESTAMP, + &state->timestamp, opt_nr * sizeof(opts[0]))) + test_error("setsockopt(TCP_TIMESTAMP)"); + } + test_sock_restore_queue(sk, TCP_RECV_QUEUE, state->in.buf, state->inq_len); + test_sock_restore_queue(sk, TCP_SEND_QUEUE, state->out.buf, state->outq_len); + if (setsockopt(sk, SOL_TCP, TCP_REPAIR_WINDOW, &state->trw, sizeof(state->trw))) + test_error("setsockopt(TCP_REPAIR_WINDOW)"); +} + +void test_ao_restore(int sk, struct tcp_ao_repair *state) +{ + if (setsockopt(sk, SOL_TCP, TCP_AO_REPAIR, state, sizeof(*state))) + test_error("setsockopt(TCP_AO_REPAIR)"); +} + +void test_sock_state_free(struct tcp_sock_state *state) +{ + free(state->out.buf); + free(state->in.buf); +} + +void test_enable_repair(int sk) +{ + int val = TCP_REPAIR_ON; + + if (setsockopt(sk, SOL_TCP, TCP_REPAIR, &val, sizeof(val))) + test_error("setsockopt(TCP_REPAIR)"); +} + +void test_disable_repair(int sk) +{ + int val = TCP_REPAIR_OFF_NO_WP; + + if (setsockopt(sk, SOL_TCP, TCP_REPAIR, &val, sizeof(val))) + test_error("setsockopt(TCP_REPAIR)"); +} + +void test_kill_sk(int sk) +{ + test_enable_repair(sk); + close(sk); +} diff --git a/tools/testing/selftests/net/tcp_ao/lib/setup.c b/tools/testing/selftests/net/tcp_ao/lib/setup.c new file mode 100644 index 00000000000000..374b27c26ebdd7 --- /dev/null +++ b/tools/testing/selftests/net/tcp_ao/lib/setup.c @@ -0,0 +1,342 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include "aolib.h" + +/* + * Can't be included in the header: it defines static variables which + * will be unique to every object. Let's include it only once here. + */ +#include "../../../kselftest.h" + +/* Prevent overriding of one thread's output by another */ +static pthread_mutex_t ksft_print_lock = PTHREAD_MUTEX_INITIALIZER; + +void __test_msg(const char *buf) +{ + pthread_mutex_lock(&ksft_print_lock); + ksft_print_msg(buf); + pthread_mutex_unlock(&ksft_print_lock); +} +void __test_ok(const char *buf) +{ + pthread_mutex_lock(&ksft_print_lock); + ksft_test_result_pass(buf); + pthread_mutex_unlock(&ksft_print_lock); +} +void __test_fail(const char *buf) +{ + pthread_mutex_lock(&ksft_print_lock); + ksft_test_result_fail(buf); + pthread_mutex_unlock(&ksft_print_lock); +} +void __test_xfail(const char *buf) +{ + pthread_mutex_lock(&ksft_print_lock); + ksft_test_result_xfail(buf); + pthread_mutex_unlock(&ksft_print_lock); +} +void __test_error(const char *buf) +{ + pthread_mutex_lock(&ksft_print_lock); + ksft_test_result_error(buf); + pthread_mutex_unlock(&ksft_print_lock); +} +void __test_skip(const char *buf) +{ + pthread_mutex_lock(&ksft_print_lock); + ksft_test_result_skip(buf); + pthread_mutex_unlock(&ksft_print_lock); +} + +static volatile int failed; +static volatile int skipped; + +void test_failed(void) +{ + failed = 1; +} + +static void test_exit(void) +{ + if (failed) { + ksft_exit_fail(); + } else if (skipped) { + /* ksft_exit_skip() is different from ksft_exit_*() */ + ksft_print_cnts(); + exit(KSFT_SKIP); + } else { + ksft_exit_pass(); + } +} + +struct dlist_t { + void (*destruct)(void); + struct dlist_t *next; +}; +static struct dlist_t *destructors_list; + +void test_add_destructor(void (*d)(void)) +{ + struct dlist_t *p; + + p = malloc(sizeof(struct dlist_t)); + if (p == NULL) + test_error("malloc() failed"); + + p->next = destructors_list; + p->destruct = d; + destructors_list = p; +} + +static void test_destructor(void) __attribute__((destructor)); +static void test_destructor(void) +{ + while (destructors_list) { + struct dlist_t *p = destructors_list->next; + + destructors_list->destruct(); + free(destructors_list); + destructors_list = p; + } + test_exit(); +} + +static void sig_int(int signo) +{ + test_error("Caught SIGINT - exiting"); +} + +int open_netns(void) +{ + const char *netns_path = "/proc/self/ns/net"; + int fd; + + fd = open(netns_path, O_RDONLY); + if (fd < 0) + test_error("open(%s)", netns_path); + return fd; +} + +int unshare_open_netns(void) +{ + if (unshare(CLONE_NEWNET) != 0) + test_error("unshare()"); + + return open_netns(); +} + +void switch_ns(int fd) +{ + if (setns(fd, CLONE_NEWNET)) + test_error("setns()"); +} + +int switch_save_ns(int new_ns) +{ + int ret = open_netns(); + + switch_ns(new_ns); + return ret; +} + +static int nsfd_outside = -1; +static int nsfd_parent = -1; +static int nsfd_child = -1; +const char veth_name[] = "ktst-veth"; + +static void init_namespaces(void) +{ + nsfd_outside = open_netns(); + nsfd_parent = unshare_open_netns(); + nsfd_child = unshare_open_netns(); +} + +static void link_init(const char *veth, int family, uint8_t prefix, + union tcp_addr addr, union tcp_addr dest) +{ + if (link_set_up(veth)) + test_error("Failed to set link up"); + if (ip_addr_add(veth, family, addr, prefix)) + test_error("Failed to add ip address"); + if (ip_route_add(veth, family, addr, dest)) + test_error("Failed to add route"); +} + +static unsigned int nr_threads = 1; + +static pthread_mutex_t sync_lock = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t sync_cond = PTHREAD_COND_INITIALIZER; +static volatile unsigned int stage_threads[2]; +static volatile unsigned int stage_nr; + +/* synchronize all threads in the same stage */ +void synchronize_threads(void) +{ + unsigned int q = stage_nr; + + pthread_mutex_lock(&sync_lock); + stage_threads[q]++; + if (stage_threads[q] == nr_threads) { + stage_nr ^= 1; + stage_threads[stage_nr] = 0; + pthread_cond_signal(&sync_cond); + } + while (stage_threads[q] < nr_threads) + pthread_cond_wait(&sync_cond, &sync_lock); + pthread_mutex_unlock(&sync_lock); +} + +__thread union tcp_addr this_ip_addr; +__thread union tcp_addr this_ip_dest; +int test_family; + +struct new_pthread_arg { + thread_fn func; + union tcp_addr my_ip; + union tcp_addr dest_ip; +}; +static void *new_pthread_entry(void *arg) +{ + struct new_pthread_arg *p = arg; + + this_ip_addr = p->my_ip; + this_ip_dest = p->dest_ip; + p->func(NULL); /* shouldn't return */ + exit(KSFT_FAIL); +} + +static void __test_skip_all(const char *msg) +{ + ksft_set_plan(1); + ksft_print_header(); + skipped = 1; + test_skip("%s", msg); + exit(KSFT_SKIP); +} + +void __test_init(unsigned int ntests, int family, unsigned int prefix, + union tcp_addr addr1, union tcp_addr addr2, + thread_fn peer1, thread_fn peer2) +{ + struct sigaction sa = { + .sa_handler = sig_int, + .sa_flags = SA_RESTART, + }; + time_t seed = time(NULL); + + sigemptyset(&sa.sa_mask); + if (sigaction(SIGINT, &sa, NULL)) + test_error("Can't set SIGINT handler"); + + test_family = family; + if (!kernel_config_has(KCONFIG_NET_NS)) + __test_skip_all(tests_skip_reason[KCONFIG_NET_NS]); + if (!kernel_config_has(KCONFIG_VETH)) + __test_skip_all(tests_skip_reason[KCONFIG_VETH]); + if (!kernel_config_has(KCONFIG_TCP_AO)) + __test_skip_all(tests_skip_reason[KCONFIG_TCP_AO]); + + ksft_set_plan(ntests); + test_print("rand seed %u", (unsigned int)seed); + srand(seed); + + + ksft_print_header(); + init_namespaces(); + + if (add_veth(veth_name, nsfd_parent, nsfd_child)) + test_error("Failed to add veth"); + + switch_ns(nsfd_child); + link_init(veth_name, family, prefix, addr2, addr1); + if (peer2) { + struct new_pthread_arg targ; + pthread_t t; + + targ.my_ip = addr2; + targ.dest_ip = addr1; + targ.func = peer2; + nr_threads++; + if (pthread_create(&t, NULL, new_pthread_entry, &targ)) + test_error("Failed to create pthread"); + } + switch_ns(nsfd_parent); + link_init(veth_name, family, prefix, addr1, addr2); + + this_ip_addr = addr1; + this_ip_dest = addr2; + peer1(NULL); + if (failed) + exit(KSFT_FAIL); + else + exit(KSFT_PASS); +} + +/* /proc/sys/net/core/optmem_max artifically limits the amount of memory + * that can be allocated with sock_kmalloc() on each socket in the system. + * It is not virtualized, so it has to written outside test namespaces. + * To be nice a test will revert optmem back to the old value. + * Keeping it simple without any file lock, which means the tests that + * need to set/increase optmem value shouldn't run in parallel. + * Also, not re-entrant. + */ +static const char *optmem_file = "/proc/sys/net/core/optmem_max"; +static size_t saved_optmem; + +size_t test_get_optmem(void) +{ + FILE *foptmem; + int old_ns; + size_t ret; + + old_ns = switch_save_ns(nsfd_outside); + foptmem = fopen(optmem_file, "r"); + if (!foptmem) + test_error("failed to open %s", optmem_file); + + if (fscanf(foptmem, "%zu", &ret) != 1) + test_error("can't read from %s", optmem_file); + fclose(foptmem); + switch_ns(old_ns); + return ret; +} + +static void __test_set_optmem(size_t new, size_t *old) +{ + FILE *foptmem; + int old_ns; + + if (old != NULL) + *old = test_get_optmem(); + + old_ns = switch_save_ns(nsfd_outside); + foptmem = fopen(optmem_file, "w"); + if (!foptmem) + test_error("failed to open %s", optmem_file); + + if (fprintf(foptmem, "%zu", new) <= 0) + test_error("can't write %zu to %s", new, optmem_file); + fclose(foptmem); + switch_ns(old_ns); +} + +static void test_revert_optmem(void) +{ + if (saved_optmem == 0) + return; + + __test_set_optmem(saved_optmem, NULL); +} + +void test_set_optmem(size_t value) +{ + if (saved_optmem == 0) { + __test_set_optmem(value, &saved_optmem); + test_add_destructor(test_revert_optmem); + } else { + __test_set_optmem(value, NULL); + } +} diff --git a/tools/testing/selftests/net/tcp_ao/lib/sock.c b/tools/testing/selftests/net/tcp_ao/lib/sock.c new file mode 100644 index 00000000000000..c75d82885a2e1a --- /dev/null +++ b/tools/testing/selftests/net/tcp_ao/lib/sock.c @@ -0,0 +1,592 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include "../../../../../include/linux/kernel.h" +#include "../../../../../include/linux/stringify.h" +#include "aolib.h" + +const unsigned int test_server_port = 7010; +int __test_listen_socket(int backlog, void *addr, size_t addr_sz) +{ + int err, sk = socket(test_family, SOCK_STREAM, IPPROTO_TCP); + long flags; + + if (sk < 0) + test_error("socket()"); + + err = setsockopt(sk, SOL_SOCKET, SO_BINDTODEVICE, veth_name, + strlen(veth_name) + 1); + if (err < 0) + test_error("setsockopt(SO_BINDTODEVICE)"); + + if (bind(sk, (struct sockaddr *)addr, addr_sz) < 0) + test_error("bind()"); + + flags = fcntl(sk, F_GETFL); + if ((flags < 0) || (fcntl(sk, F_SETFL, flags | O_NONBLOCK) < 0)) + test_error("fcntl()"); + + if (listen(sk, backlog)) + test_error("listen()"); + + return sk; +} + +int test_wait_fd(int sk, time_t sec, bool write) +{ + struct timeval tv = { .tv_sec = sec }; + struct timeval *ptv = NULL; + fd_set fds, efds; + int ret; + socklen_t slen = sizeof(ret); + + FD_ZERO(&fds); + FD_SET(sk, &fds); + FD_ZERO(&efds); + FD_SET(sk, &efds); + + if (sec) + ptv = &tv; + + errno = 0; + if (write) + ret = select(sk + 1, NULL, &fds, &efds, ptv); + else + ret = select(sk + 1, &fds, NULL, &efds, ptv); + if (ret < 0) + return -errno; + if (ret == 0) { + errno = ETIMEDOUT; + return -ETIMEDOUT; + } + + if (getsockopt(sk, SOL_SOCKET, SO_ERROR, &ret, &slen) || ret) + return -ret; + return 0; +} + +int __test_connect_socket(int sk, const char *device, + void *addr, size_t addr_sz, time_t timeout) +{ + long flags; + int err; + + if (device != NULL) { + err = setsockopt(sk, SOL_SOCKET, SO_BINDTODEVICE, device, + strlen(device) + 1); + if (err < 0) + test_error("setsockopt(SO_BINDTODEVICE, %s)", device); + } + + if (!timeout) { + err = connect(sk, addr, addr_sz); + if (err) { + err = -errno; + goto out; + } + return 0; + } + + flags = fcntl(sk, F_GETFL); + if ((flags < 0) || (fcntl(sk, F_SETFL, flags | O_NONBLOCK) < 0)) + test_error("fcntl()"); + + if (connect(sk, addr, addr_sz) < 0) { + if (errno != EINPROGRESS) { + err = -errno; + goto out; + } + if (timeout < 0) + return sk; + err = test_wait_fd(sk, timeout, 1); + if (err) + goto out; + } + return sk; + +out: + close(sk); + return err; +} + +int __test_set_md5(int sk, void *addr, size_t addr_sz, uint8_t prefix, + int vrf, const char *password) +{ + size_t pwd_len = strlen(password); + struct tcp_md5sig md5sig = {}; + + md5sig.tcpm_keylen = pwd_len; + memcpy(md5sig.tcpm_key, password, pwd_len); + md5sig.tcpm_flags = TCP_MD5SIG_FLAG_PREFIX; + md5sig.tcpm_prefixlen = prefix; + if (vrf >= 0) { + md5sig.tcpm_flags |= TCP_MD5SIG_FLAG_IFINDEX; + md5sig.tcpm_ifindex = (uint8_t)vrf; + } + memcpy(&md5sig.tcpm_addr, addr, addr_sz); + + errno = 0; + return setsockopt(sk, IPPROTO_TCP, TCP_MD5SIG_EXT, + &md5sig, sizeof(md5sig)); +} + + +int test_prepare_key_sockaddr(struct tcp_ao_add *ao, const char *alg, + void *addr, size_t addr_sz, bool set_current, bool set_rnext, + uint8_t prefix, uint8_t vrf, uint8_t sndid, uint8_t rcvid, + uint8_t maclen, uint8_t keyflags, + uint8_t keylen, const char *key) +{ + memset(ao, 0, sizeof(struct tcp_ao_add)); + + ao->set_current = !!set_current; + ao->set_rnext = !!set_rnext; + ao->prefix = prefix; + ao->sndid = sndid; + ao->rcvid = rcvid; + ao->maclen = maclen; + ao->keyflags = keyflags; + ao->keylen = keylen; + ao->ifindex = vrf; + + memcpy(&ao->addr, addr, addr_sz); + + if (strlen(alg) > 64) + return -ENOBUFS; + strncpy(ao->alg_name, alg, 64); + + memcpy(ao->key, key, + (keylen > TCP_AO_MAXKEYLEN) ? TCP_AO_MAXKEYLEN : keylen); + return 0; +} + +static int test_get_ao_keys_nr(int sk) +{ + struct tcp_ao_getsockopt tmp = {}; + socklen_t tmp_sz = sizeof(tmp); + int ret; + + tmp.nkeys = 1; + tmp.get_all = 1; + + ret = getsockopt(sk, IPPROTO_TCP, TCP_AO_GET_KEYS, &tmp, &tmp_sz); + if (ret) + return -errno; + return (int)tmp.nkeys; +} + +int test_get_one_ao(int sk, struct tcp_ao_getsockopt *out, + void *addr, size_t addr_sz, uint8_t prefix, + uint8_t sndid, uint8_t rcvid) +{ + struct tcp_ao_getsockopt tmp = {}; + socklen_t tmp_sz = sizeof(tmp); + int ret; + + memcpy(&tmp.addr, addr, addr_sz); + tmp.prefix = prefix; + tmp.sndid = sndid; + tmp.rcvid = rcvid; + tmp.nkeys = 1; + + ret = getsockopt(sk, IPPROTO_TCP, TCP_AO_GET_KEYS, &tmp, &tmp_sz); + if (ret) + return ret; + if (tmp.nkeys != 1) + return -E2BIG; + *out = tmp; + return 0; +} + +int test_get_ao_info(int sk, struct tcp_ao_info_opt *out) +{ + socklen_t sz = sizeof(*out); + + out->reserved = 0; + out->reserved2 = 0; + if (getsockopt(sk, IPPROTO_TCP, TCP_AO_INFO, out, &sz)) + return -errno; + if (sz != sizeof(*out)) + return -EMSGSIZE; + return 0; +} + +int test_set_ao_info(int sk, struct tcp_ao_info_opt *in) +{ + socklen_t sz = sizeof(*in); + + in->reserved = 0; + in->reserved2 = 0; + if (setsockopt(sk, IPPROTO_TCP, TCP_AO_INFO, in, sz)) + return -errno; + return 0; +} + +int test_cmp_getsockopt_setsockopt(const struct tcp_ao_add *a, + const struct tcp_ao_getsockopt *b) +{ + bool is_kdf_aes_128_cmac = false; + bool is_cmac_aes = false; + + if (!strcmp("cmac(aes128)", a->alg_name)) { + is_kdf_aes_128_cmac = (a->keylen != 16); + is_cmac_aes = true; + } + +#define __cmp_ao(member) \ +do { \ + if (b->member != a->member) { \ + test_fail("getsockopt(): " __stringify(member) " %u != %u", \ + b->member, a->member); \ + return -1; \ + } \ +} while(0) + __cmp_ao(sndid); + __cmp_ao(rcvid); + __cmp_ao(prefix); + __cmp_ao(keyflags); + __cmp_ao(ifindex); + if (a->maclen) { + __cmp_ao(maclen); + } else if (b->maclen != 12) { + test_fail("getsockopt(): expected default maclen 12, but it's %u", + b->maclen); + return -1; + } + if (!is_kdf_aes_128_cmac) { + __cmp_ao(keylen); + } else if (b->keylen != 16) { + test_fail("getsockopt(): expected keylen 16 for cmac(aes128), but it's %u", + b->keylen); + return -1; + } +#undef __cmp_ao + if (!is_kdf_aes_128_cmac && memcmp(b->key, a->key, a->keylen)) { + test_fail("getsockopt(): returned key is different `%s' != `%s'", + b->key, a->key); + return -1; + } + if (memcmp(&b->addr, &a->addr, sizeof(b->addr))) { + test_fail("getsockopt(): returned address is different"); + return -1; + } + if (!is_cmac_aes && strcmp(b->alg_name, a->alg_name)) { + test_fail("getsockopt(): returned algorithm %s is different than %s", b->alg_name, a->alg_name); + return -1; + } + if (is_cmac_aes && strcmp(b->alg_name, "cmac(aes)")) { + test_fail("getsockopt(): returned algorithm %s is different than cmac(aes)", b->alg_name); + return -1; + } + /* For a established key rotation test don't add a key with + * set_current = 1, as it's likely to change by peer's request; + * rather use setsockopt(TCP_AO_INFO) + */ + if (a->set_current != b->is_current) { + test_fail("getsockopt(): returned key is not Current_key"); + return -1; + } + if (a->set_rnext != b->is_rnext) { + test_fail("getsockopt(): returned key is not RNext_key"); + return -1; + } + + return 0; +} + +int test_cmp_getsockopt_setsockopt_ao(const struct tcp_ao_info_opt *a, + const struct tcp_ao_info_opt *b) +{ + /* No check for ::current_key, as it may change by the peer */ + if (a->ao_required != b->ao_required) { + test_fail("getsockopt(): returned ao doesn't have ao_required"); + return -1; + } + if (a->accept_icmps != b->accept_icmps) { + test_fail("getsockopt(): returned ao doesn't accept ICMPs"); + return -1; + } + if (a->set_rnext && a->rnext != b->rnext) { + test_fail("getsockopt(): RNext KeyID has changed"); + return -1; + } +#define __cmp_cnt(member) \ +do { \ + if (b->member != a->member) { \ + test_fail("getsockopt(): " __stringify(member) " %llu != %llu", \ + b->member, a->member); \ + return -1; \ + } \ +} while(0) + if (a->set_counters) { + __cmp_cnt(pkt_good); + __cmp_cnt(pkt_bad); + __cmp_cnt(pkt_key_not_found); + __cmp_cnt(pkt_ao_required); + __cmp_cnt(pkt_dropped_icmp); + } +#undef __cmp_cnt + return 0; +} + +int test_get_tcp_ao_counters(int sk, struct tcp_ao_counters *out) +{ + struct tcp_ao_getsockopt *key_dump; + socklen_t key_dump_sz = sizeof(*key_dump); + struct tcp_ao_info_opt info = {}; + bool c1, c2, c3, c4, c5; + struct netstat *ns; + int err, nr_keys; + + memset(out, 0, sizeof(*out)); + + /* per-netns */ + ns = netstat_read(); + out->netns_ao_good = netstat_get(ns, "TCPAOGood", &c1); + out->netns_ao_bad = netstat_get(ns, "TCPAOBad", &c2); + out->netns_ao_key_not_found = netstat_get(ns, "TCPAOKeyNotFound", &c3); + out->netns_ao_required = netstat_get(ns, "TCPAORequired", &c4); + out->netns_ao_dropped_icmp = netstat_get(ns, "TCPAODroppedIcmps", &c5); + netstat_free(ns); + if (c1 || c2 || c3 || c4 || c5) + return -EOPNOTSUPP; + + err = test_get_ao_info(sk, &info); + if (err) + return err; + + /* per-socket */ + out->ao_info_pkt_good = info.pkt_good; + out->ao_info_pkt_bad = info.pkt_bad; + out->ao_info_pkt_key_not_found = info.pkt_key_not_found; + out->ao_info_pkt_ao_required = info.pkt_ao_required; + out->ao_info_pkt_dropped_icmp = info.pkt_dropped_icmp; + + /* per-key */ + nr_keys = test_get_ao_keys_nr(sk); + if (nr_keys < 0) + return nr_keys; + if (nr_keys == 0) + test_error("test_get_ao_keys_nr() == 0"); + out->nr_keys = (size_t)nr_keys; + key_dump = calloc(nr_keys, key_dump_sz); + if (!key_dump) + return -errno; + + key_dump[0].nkeys = nr_keys; + key_dump[0].get_all = 1; + key_dump[0].get_all = 1; + err = getsockopt(sk, IPPROTO_TCP, TCP_AO_GET_KEYS, + key_dump, &key_dump_sz); + if (err) { + free(key_dump); + return -errno; + } + + out->key_cnts = calloc(nr_keys, sizeof(out->key_cnts[0])); + if (!out->key_cnts) { + free(key_dump); + return -errno; + } + + while (nr_keys--) { + out->key_cnts[nr_keys].sndid = key_dump[nr_keys].sndid; + out->key_cnts[nr_keys].rcvid = key_dump[nr_keys].rcvid; + out->key_cnts[nr_keys].pkt_good = key_dump[nr_keys].pkt_good; + out->key_cnts[nr_keys].pkt_bad = key_dump[nr_keys].pkt_bad; + } + free(key_dump); + + return 0; +} + +int __test_tcp_ao_counters_cmp(const char *tst_name, + struct tcp_ao_counters *before, + struct tcp_ao_counters *after, + test_cnt expected) +{ +#define __cmp_ao(cnt, expecting_inc) \ +do { \ + if (before->cnt > after->cnt) { \ + test_fail("%s: Decreased counter " __stringify(cnt) " %" PRIu64 " > %" PRIu64, \ + tst_name ?: "", before->cnt, after->cnt); \ + return -1; \ + } \ + if ((before->cnt != after->cnt) != (expecting_inc)) { \ + test_fail("%s: Counter " __stringify(cnt) " was %sexpected to increase %" PRIu64 " => %" PRIu64, \ + tst_name ?: "", (expecting_inc) ? "" : "not ", \ + before->cnt, after->cnt); \ + return -1; \ + } \ +} while(0) + + errno = 0; + /* per-netns */ + __cmp_ao(netns_ao_good, !!(expected & TEST_CNT_NS_GOOD)); + __cmp_ao(netns_ao_bad, !!(expected & TEST_CNT_NS_BAD)); + __cmp_ao(netns_ao_key_not_found, + !!(expected & TEST_CNT_NS_KEY_NOT_FOUND)); + __cmp_ao(netns_ao_required, !!(expected & TEST_CNT_NS_AO_REQUIRED)); + __cmp_ao(netns_ao_dropped_icmp, + !!(expected & TEST_CNT_NS_DROPPED_ICMP)); + /* per-socket */ + __cmp_ao(ao_info_pkt_good, !!(expected & TEST_CNT_SOCK_GOOD)); + __cmp_ao(ao_info_pkt_bad, !!(expected & TEST_CNT_SOCK_BAD)); + __cmp_ao(ao_info_pkt_key_not_found, + !!(expected & TEST_CNT_SOCK_KEY_NOT_FOUND)); + __cmp_ao(ao_info_pkt_ao_required, !!(expected & TEST_CNT_SOCK_AO_REQUIRED)); + __cmp_ao(ao_info_pkt_dropped_icmp, + !!(expected & TEST_CNT_SOCK_DROPPED_ICMP)); + return 0; +#undef __cmp_ao +} + +int test_tcp_ao_key_counters_cmp(const char *tst_name, + struct tcp_ao_counters *before, + struct tcp_ao_counters *after, + test_cnt expected, + int sndid, int rcvid) +{ + size_t i; +#define __cmp_ao(i, cnt, expecting_inc) \ +do { \ + if (before->key_cnts[i].cnt > after->key_cnts[i].cnt) { \ + test_fail("%s: Decreased counter " __stringify(cnt) " %" PRIu64 " > %" PRIu64 " for key %u:%u", \ + tst_name ?: "", before->key_cnts[i].cnt, \ + after->key_cnts[i].cnt, \ + before->key_cnts[i].sndid, \ + before->key_cnts[i].rcvid); \ + return -1; \ + } \ + if ((before->key_cnts[i].cnt != after->key_cnts[i].cnt) != (expecting_inc)) { \ + test_fail("%s: Counter " __stringify(cnt) " was %sexpected to increase %" PRIu64 " => %" PRIu64 " for key %u:%u", \ + tst_name ?: "", (expecting_inc) ? "" : "not ",\ + before->key_cnts[i].cnt, \ + after->key_cnts[i].cnt, \ + before->key_cnts[i].sndid, \ + before->key_cnts[i].rcvid); \ + return -1; \ + } \ +} while(0) + + if (before->nr_keys != after->nr_keys) { + test_fail("%s: Keys changed on the socket %zu != %zu", + tst_name, before->nr_keys, after->nr_keys); + return -1; + } + + /* per-key */ + i = before->nr_keys; + while (i--) { + if (sndid >= 0 && before->key_cnts[i].sndid != sndid) + continue; + if (rcvid >= 0 && before->key_cnts[i].rcvid != rcvid) + continue; + __cmp_ao(i, pkt_good, !!(expected & TEST_CNT_KEY_GOOD)); + __cmp_ao(i, pkt_bad, !!(expected & TEST_CNT_KEY_BAD)); + } + return 0; +#undef __cmp_ao +} + +void test_tcp_ao_counters_free(struct tcp_ao_counters *cnts) +{ + free(cnts->key_cnts); +} + +#define TEST_BUF_SIZE 4096 +ssize_t test_server_run(int sk, ssize_t quota, time_t timeout_sec) +{ + ssize_t total = 0; + + do { + char buf[TEST_BUF_SIZE]; + ssize_t bytes, sent; + int ret; + + ret = test_wait_fd(sk, timeout_sec, 0); + if (ret) + return ret; + + bytes = recv(sk, buf, sizeof(buf), 0); + + if (bytes < 0) + test_error("recv(): %zd", bytes); + if (bytes == 0) + break; + + ret = test_wait_fd(sk, timeout_sec, 1); + if (ret) + return ret; + + sent = send(sk, buf, bytes, 0); + if (sent == 0) + break; + if (sent != bytes) + test_error("send()"); + total += bytes; + } while (!quota || total < quota); + + return total; +} + +ssize_t test_client_loop(int sk, char *buf, size_t buf_sz, + const size_t msg_len, time_t timeout_sec) +{ + char msg[msg_len]; + int nodelay = 1; + size_t i; + + if (setsockopt(sk, IPPROTO_TCP, TCP_NODELAY, &nodelay, sizeof(nodelay))) + test_error("setsockopt(TCP_NODELAY)"); + + for (i = 0; i < buf_sz; i += min(msg_len, buf_sz - i)) { + size_t sent, bytes = min(msg_len, buf_sz - i); + int ret; + + ret = test_wait_fd(sk, timeout_sec, 1); + if (ret) + return ret; + + sent = send(sk, buf + i, bytes, 0); + if (sent == 0) + break; + if (sent != bytes) + test_error("send()"); + + bytes = 0; + do { + ssize_t got; + + ret = test_wait_fd(sk, timeout_sec, 0); + if (ret) + return ret; + + got = recv(sk, msg + bytes, sizeof(msg) - bytes, 0); + if (got <= 0) + return i; + bytes += got; + } while (bytes < sent); + if (bytes > sent) + test_error("recv(): %zd > %zd", bytes, sent); + if (memcmp(buf + i, msg, bytes) != 0) { + test_fail("received message differs"); + return -1; + } + } + return i; +} + +int test_client_verify(int sk, const size_t msg_len, const size_t nr, + time_t timeout_sec) +{ + size_t buf_sz = msg_len * nr; + char *buf = alloca(buf_sz); + + randomize_buffer(buf, buf_sz); + if (test_client_loop(sk, buf, buf_sz, msg_len, timeout_sec) != buf_sz) + return -1; + return 0; +} diff --git a/tools/testing/selftests/net/tcp_ao/lib/utils.c b/tools/testing/selftests/net/tcp_ao/lib/utils.c new file mode 100644 index 00000000000000..372daca525f5d0 --- /dev/null +++ b/tools/testing/selftests/net/tcp_ao/lib/utils.c @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "aolib.h" +#include + +void randomize_buffer(void *buf, size_t buflen) +{ + int *p = (int *)buf; + size_t words = buflen / sizeof(int); + size_t leftover = buflen % sizeof(int); + + if (!buflen) + return; + + while (words--) + *p++ = rand(); + + if (leftover) { + int tmp = rand(); + + memcpy(buf + buflen - leftover, &tmp, leftover); + } +} + +const struct sockaddr_in6 addr_any6 = { + .sin6_family = AF_INET6, +}; + +const struct sockaddr_in addr_any4 = { + .sin_family = AF_INET, +}; diff --git a/tools/testing/selftests/net/tcp_ao/restore.c b/tools/testing/selftests/net/tcp_ao/restore.c new file mode 100644 index 00000000000000..8fdc808df32563 --- /dev/null +++ b/tools/testing/selftests/net/tcp_ao/restore.c @@ -0,0 +1,236 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Author: Dmitry Safonov */ +/* This is over-simplified TCP_REPAIR for TCP_ESTABLISHED sockets + * It tests that TCP-AO enabled connection can be restored. + * For the proper socket repair see: + * https://github.com/checkpoint-restore/criu/blob/criu-dev/soccr/soccr.h + */ +#include +#include "aolib.h" + +const size_t nr_packets = 20; +const size_t msg_len = 100; +const size_t quota = nr_packets * msg_len; +#define fault(type) (inj == FAULT_ ## type) + +static void try_server_run(const char *tst_name, unsigned int port, + fault_t inj, test_cnt cnt_expected) +{ + const char *cnt_name = "TCPAOGood"; + struct tcp_ao_counters ao1, ao2; + uint64_t before_cnt, after_cnt; + int sk, lsk; + time_t timeout; + ssize_t bytes; + + if (fault(TIMEOUT)) + cnt_name = "TCPAOBad"; + lsk = test_listen_socket(this_ip_addr, port, 1); + + if (test_add_key(lsk, DEFAULT_TEST_PASSWORD, this_ip_dest, -1, 100, 100)) + test_error("setsockopt(TCP_AO_ADD_KEY)"); + synchronize_threads(); /* 1: MKT added => connect() */ + + if (test_wait_fd(lsk, TEST_TIMEOUT_SEC, 0)) + test_error("test_wait_fd()"); + + sk = accept(lsk, NULL, NULL); + if (sk < 0) + test_error("accept()"); + + synchronize_threads(); /* 2: accepted => send data */ + close(lsk); + + bytes = test_server_run(sk, quota, TEST_TIMEOUT_SEC); + if (bytes != quota) { + test_fail("%s: server served: %zd", tst_name, bytes); + goto out; + } + + before_cnt = netstat_get_one(cnt_name, NULL); + if (test_get_tcp_ao_counters(sk, &ao1)) + test_error("test_get_tcp_ao_counters()"); + + timeout = fault(TIMEOUT) ? TEST_RETRANSMIT_SEC : TEST_TIMEOUT_SEC; + bytes = test_server_run(sk, quota, timeout); + if (fault(TIMEOUT)) { + if (bytes > 0) + test_fail("%s: server served: %zd", tst_name, bytes); + else + test_ok("%s: server couldn't serve", tst_name); + } else { + if (bytes != quota) + test_fail("%s: server served: %zd", tst_name, bytes); + else + test_ok("%s: server alive", tst_name); + } + if (test_get_tcp_ao_counters(sk, &ao2)) + test_error("test_get_tcp_ao_counters()"); + after_cnt = netstat_get_one(cnt_name, NULL); + + test_tcp_ao_counters_cmp(tst_name, &ao1, &ao2, cnt_expected); + + if (after_cnt <= before_cnt) { + test_fail("%s: %s counter did not increase: %zu <= %zu", + tst_name, cnt_name, after_cnt, before_cnt); + } else { + test_ok("%s: counter %s increased %zu => %zu", + tst_name, cnt_name, before_cnt, after_cnt); + } + + /* + * Before close() as that will send FIN and move the peer in TCP_CLOSE + * and that will prevent reading AO counters from the peer's socket. + */ + synchronize_threads(); /* 3: verified => closed */ +out: + close(sk); +} + +static void *server_fn(void *arg) +{ + unsigned int port = test_server_port; + + try_server_run("TCP-AO migrate to another socket", port++, + 0, TEST_CNT_GOOD); + try_server_run("TCP-AO with wrong send ISN", port++, + FAULT_TIMEOUT, TEST_CNT_BAD); + try_server_run("TCP-AO with wrong receive ISN", port++, + FAULT_TIMEOUT, TEST_CNT_BAD); + try_server_run("TCP-AO with wrong send SEQ ext number", port++, + FAULT_TIMEOUT, TEST_CNT_BAD); + try_server_run("TCP-AO with wrong receive SEQ ext number", port++, + FAULT_TIMEOUT, TEST_CNT_NS_BAD | TEST_CNT_GOOD); + + synchronize_threads(); /* don't race to exit: client exits */ + return NULL; +} + +static void test_get_sk_checkpoint(unsigned int server_port, sockaddr_af *saddr, + struct tcp_sock_state *img, + struct tcp_ao_repair *ao_img) +{ + int sk; + + sk = socket(test_family, SOCK_STREAM, IPPROTO_TCP); + if (sk < 0) + test_error("socket()"); + + if (test_add_key(sk, DEFAULT_TEST_PASSWORD, this_ip_dest, -1, 100, 100)) + test_error("setsockopt(TCP_AO_ADD_KEY)"); + + synchronize_threads(); /* 1: MKT added => connect() */ + if (test_connect_socket(sk, this_ip_dest, server_port) <= 0) + test_error("failed to connect()"); + + synchronize_threads(); /* 2: accepted => send data */ + if (test_client_verify(sk, msg_len, nr_packets, TEST_TIMEOUT_SEC)) + test_fail("pre-migrate verify failed"); + + test_enable_repair(sk); + test_sock_checkpoint(sk, img, saddr); + test_ao_checkpoint(sk, ao_img); + test_kill_sk(sk); +} + +static void test_sk_restore(const char *tst_name, unsigned int server_port, + sockaddr_af *saddr, struct tcp_sock_state *img, + struct tcp_ao_repair *ao_img, + fault_t inj, test_cnt cnt_expected) +{ + const char *cnt_name = "TCPAOGood"; + struct tcp_ao_counters ao1, ao2; + uint64_t before_cnt, after_cnt; + time_t timeout; + int sk; + + if (fault(TIMEOUT)) + cnt_name = "TCPAOBad"; + + before_cnt = netstat_get_one(cnt_name, NULL); + sk = socket(test_family, SOCK_STREAM, IPPROTO_TCP); + if (sk < 0) + test_error("socket()"); + + test_enable_repair(sk); + test_sock_restore(sk, img, saddr, this_ip_dest, server_port); + if (test_add_repaired_key(sk, DEFAULT_TEST_PASSWORD, 0, this_ip_dest, -1, 100, 100)) + test_error("setsockopt(TCP_AO_ADD_KEY)"); + test_ao_restore(sk, ao_img); + + if (test_get_tcp_ao_counters(sk, &ao1)) + test_error("test_get_tcp_ao_counters()"); + + test_disable_repair(sk); + test_sock_state_free(img); + + timeout = fault(TIMEOUT) ? TEST_RETRANSMIT_SEC : TEST_TIMEOUT_SEC; + if (test_client_verify(sk, msg_len, nr_packets, timeout)) { + if (fault(TIMEOUT)) + test_ok("%s: post-migrate connection is broken", tst_name); + else + test_fail("%s: post-migrate connection is working", tst_name); + } else { + if (fault(TIMEOUT)) + test_fail("%s: post-migrate connection still working", tst_name); + else + test_ok("%s: post-migrate connection is alive", tst_name); + } + if (test_get_tcp_ao_counters(sk, &ao2)) + test_error("test_get_tcp_ao_counters()"); + after_cnt = netstat_get_one(cnt_name, NULL); + + test_tcp_ao_counters_cmp(tst_name, &ao1, &ao2, cnt_expected); + + if (after_cnt <= before_cnt) { + test_fail("%s: %s counter did not increase: %zu <= %zu", + tst_name, cnt_name, after_cnt, before_cnt); + } else { + test_ok("%s: counter %s increased %zu => %zu", + tst_name, cnt_name, before_cnt, after_cnt); + } + synchronize_threads(); /* 3: verified => closed */ + close(sk); +} + +static void *client_fn(void *arg) +{ + unsigned int port = test_server_port; + struct tcp_sock_state tcp_img; + struct tcp_ao_repair ao_img; + sockaddr_af saddr; + + test_get_sk_checkpoint(port, &saddr, &tcp_img, &ao_img); + test_sk_restore("TCP-AO migrate to another socket", port++, + &saddr, &tcp_img, &ao_img, 0, TEST_CNT_GOOD); + + test_get_sk_checkpoint(port, &saddr, &tcp_img, &ao_img); + ao_img.snt_isn += 1; + test_sk_restore("TCP-AO with wrong send ISN", port++, + &saddr, &tcp_img, &ao_img, FAULT_TIMEOUT, TEST_CNT_BAD); + + test_get_sk_checkpoint(port, &saddr, &tcp_img, &ao_img); + ao_img.rcv_isn += 1; + test_sk_restore("TCP-AO with wrong receive ISN", port++, + &saddr, &tcp_img, &ao_img, FAULT_TIMEOUT, TEST_CNT_BAD); + + test_get_sk_checkpoint(port, &saddr, &tcp_img, &ao_img); + ao_img.snd_sne += 1; + test_sk_restore("TCP-AO with wrong send SEQ ext number", port++, + &saddr, &tcp_img, &ao_img, FAULT_TIMEOUT, + TEST_CNT_NS_BAD | TEST_CNT_GOOD); + + test_get_sk_checkpoint(port, &saddr, &tcp_img, &ao_img); + ao_img.rcv_sne += 1; + test_sk_restore("TCP-AO with wrong receive SEQ ext number", port++, + &saddr, &tcp_img, &ao_img, FAULT_TIMEOUT, + TEST_CNT_NS_GOOD | TEST_CNT_BAD); + + return NULL; +} + +int main(int argc, char *argv[]) +{ + test_init(20, server_fn, client_fn); + return 0; +} diff --git a/tools/testing/selftests/net/tcp_ao/rst.c b/tools/testing/selftests/net/tcp_ao/rst.c new file mode 100644 index 00000000000000..ac06009a7f5f65 --- /dev/null +++ b/tools/testing/selftests/net/tcp_ao/rst.c @@ -0,0 +1,415 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Author: Dmitry Safonov */ +#include +#include "../../../../include/linux/kernel.h" +#include "aolib.h" + +const size_t quota = 1000; +/* + * Backlog == 0 means 1 connection in queue, see: + * commit 64a146513f8f ("[NET]: Revert incorrect accept queue...") + */ +const unsigned int backlog; + +static void netstats_check(struct netstat *before, struct netstat *after, + char *msg) +{ + uint64_t before_cnt, after_cnt; + + before_cnt = netstat_get(before, "TCPAORequired", NULL); + after_cnt = netstat_get(after, "TCPAORequired", NULL); + if (after_cnt > before_cnt) + test_fail("Segments without AO sign (%s): %" PRIu64 " => %" PRIu64, + msg, before_cnt, after_cnt); + else + test_ok("No segments without AO sign (%s)", msg); + + before_cnt = netstat_get(before, "TCPAOGood", NULL); + after_cnt = netstat_get(after, "TCPAOGood", NULL); + if (after_cnt <= before_cnt) + test_fail("Signed AO segments (%s): %" PRIu64 " => %" PRIu64, + msg, before_cnt, after_cnt); + else + test_ok("Signed AO segments (%s): %" PRIu64 " => %" PRIu64, + msg, before_cnt, after_cnt); + + before_cnt = netstat_get(before, "TCPAOBad", NULL); + after_cnt = netstat_get(after, "TCPAOBad", NULL); + if (after_cnt > before_cnt) + test_fail("Segments with bad AO sign (%s): %" PRIu64 " => %" PRIu64, + msg, before_cnt, after_cnt); + else + test_ok("No segments with bad AO sign (%s)", msg); +} + +/* + * Another way to send RST, but not through tcp_v{4,6}_send_reset() + * is tcp_send_active_reset(), that is not in reply to inbound segment, + * but rather active send. It uses tcp_transmit_skb(), so that should + * work, but as it also sends RST - nice that it can be covered as well. + */ +static void close_forced(int sk) +{ + struct linger sl; + + sl.l_onoff = 1; + sl.l_linger = 0; + if (setsockopt(sk, SOL_SOCKET, SO_LINGER, &sl, sizeof(sl))) + test_error("setsockopt(SO_LINGER)"); + close(sk); +} + +static int test_wait_for_exception(int sk, time_t sec) +{ + struct timeval tv = { .tv_sec = sec }; + struct timeval *ptv = NULL; + fd_set efds; + int ret; + + FD_ZERO(&efds); + FD_SET(sk, &efds); + + if (sec) + ptv = &tv; + + errno = 0; + ret = select(sk + 1, NULL, NULL, &efds, ptv); + if (ret < 0) + return -errno; + return ret ? sk : 0; +} + +static void test_server_active_rst(unsigned int port) +{ + struct tcp_ao_counters cnt1, cnt2; + ssize_t bytes; + int sk, lsk; + + lsk = test_listen_socket(this_ip_addr, port, backlog); + if (test_add_key(lsk, DEFAULT_TEST_PASSWORD, this_ip_dest, -1, 100, 100)) + test_error("setsockopt(TCP_AO_ADD_KEY)"); + if (test_get_tcp_ao_counters(lsk, &cnt1)) + test_error("test_get_tcp_ao_counters()"); + + synchronize_threads(); /* 1: MKT added */ + if (test_wait_fd(lsk, TEST_TIMEOUT_SEC, 0)) + test_error("test_wait_fd()"); + + sk = accept(lsk, NULL, NULL); + if (sk < 0) + test_error("accept()"); + + synchronize_threads(); /* 2: connection accept()ed, another queued */ + if (test_get_tcp_ao_counters(lsk, &cnt2)) + test_error("test_get_tcp_ao_counters()"); + + synchronize_threads(); /* 3: close listen socket */ + close(lsk); + bytes = test_server_run(sk, quota, 0); + if (bytes != quota) + test_error("servered only %zd bytes", bytes); + else + test_ok("servered %zd bytes", bytes); + + synchronize_threads(); /* 4: finishing up */ + close_forced(sk); + + synchronize_threads(); /* 5: closed active sk */ + + synchronize_threads(); /* 6: counters checks */ + if (test_tcp_ao_counters_cmp("active RST server", &cnt1, &cnt2, TEST_CNT_GOOD)) + test_fail("MKT counters (server) have not only good packets"); + else + test_ok("MKT counters are good on server"); +} + +static void test_server_passive_rst(unsigned int port) +{ + struct tcp_ao_counters ao1, ao2; + int sk, lsk; + ssize_t bytes; + + lsk = test_listen_socket(this_ip_addr, port, 1); + + if (test_add_key(lsk, DEFAULT_TEST_PASSWORD, this_ip_dest, -1, 100, 100)) + test_error("setsockopt(TCP_AO_ADD_KEY)"); + + synchronize_threads(); /* 1: MKT added => connect() */ + if (test_wait_fd(lsk, TEST_TIMEOUT_SEC, 0)) + test_error("test_wait_fd()"); + + sk = accept(lsk, NULL, NULL); + if (sk < 0) + test_error("accept()"); + + synchronize_threads(); /* 2: accepted => send data */ + close(lsk); + if (test_get_tcp_ao_counters(sk, &ao1)) + test_error("test_get_tcp_ao_counters()"); + + bytes = test_server_run(sk, quota, TEST_TIMEOUT_SEC); + if (bytes != quota) { + if (bytes > 0) + test_fail("server served: %zd", bytes); + else + test_fail("server returned %zd", bytes); + } + + synchronize_threads(); /* 3: chekpoint/restore the connection */ + if (test_get_tcp_ao_counters(sk, &ao2)) + test_error("test_get_tcp_ao_counters()"); + + synchronize_threads(); /* 4: terminate server + send more on client */ + bytes = test_server_run(sk, quota, TEST_RETRANSMIT_SEC); + close(sk); + test_tcp_ao_counters_cmp("passive RST server", &ao1, &ao2, TEST_CNT_GOOD); + + synchronize_threads(); /* 5: verified => closed */ + close(sk); +} + +static void *server_fn(void *arg) +{ + struct netstat *ns_before, *ns_after; + unsigned int port = test_server_port; + + ns_before = netstat_read(); + + test_server_active_rst(port++); + test_server_passive_rst(port++); + + ns_after = netstat_read(); + netstats_check(ns_before, ns_after, "server"); + netstat_free(ns_after); + netstat_free(ns_before); + synchronize_threads(); /* exit */ + + synchronize_threads(); /* don't race to exit() - client exits */ + return NULL; +} + +static int test_wait_fds(int sk[], size_t nr, bool is_writable[], + ssize_t wait_for, time_t sec) +{ + struct timeval tv = { .tv_sec = sec }; + struct timeval *ptv = NULL; + fd_set left; + size_t i; + int ret; + + FD_ZERO(&left); + for (i = 0; i < nr; i++) { + FD_SET(sk[i], &left); + if (is_writable) + is_writable[i] = false; + } + + if (sec) + ptv = &tv; + + do { + bool is_empty = true; + fd_set fds, efds; + int nfd = 0; + + FD_ZERO(&fds); + FD_ZERO(&efds); + for (i = 0; i < nr; i++) { + if (!FD_ISSET(sk[i], &left)) + continue; + + if (sk[i] > nfd) + nfd = sk[i]; + + FD_SET(sk[i], &fds); + FD_SET(sk[i], &efds); + is_empty = false; + } + if (is_empty) + return -ENOENT; + + errno = 0; + ret = select(nfd + 1, NULL, &fds, &efds, ptv); + if (ret < 0) + return -errno; + if (!ret) + return -ETIMEDOUT; + for (i = 0; i < nr; i++) { + if (FD_ISSET(sk[i], &fds)) { + if (is_writable) + is_writable[i] = true; + FD_CLR(sk[i], &left); + wait_for--; + continue; + } + if (FD_ISSET(sk[i], &efds)) { + FD_CLR(sk[i], &left); + wait_for--; + } + } + } while (wait_for > 0); + + return 0; +} + +static void test_client_active_rst(unsigned int port) +{ + /* one in queue, another accept()ed */ + unsigned int wait_for = backlog + 2; + int i, sk[3], err; + bool is_writable[ARRAY_SIZE(sk)] = {false}; + unsigned int last = ARRAY_SIZE(sk) - 1; + + for (i = 0; i < ARRAY_SIZE(sk); i++) { + sk[i] = socket(test_family, SOCK_STREAM, IPPROTO_TCP); + if (sk[i] < 0) + test_error("socket()"); + if (test_add_key(sk[i], DEFAULT_TEST_PASSWORD, + this_ip_dest, -1, 100, 100)) + test_error("setsockopt(TCP_AO_ADD_KEY)"); + } + + synchronize_threads(); /* 1: MKT added */ + for (i = 0; i < last; i++) { + err = _test_connect_socket(sk[i], this_ip_dest, port, + (i == 0) ? TEST_TIMEOUT_SEC : -1); + + if (err < 0) + test_error("failed to connect()"); + } + + synchronize_threads(); /* 2: connection accept()ed, another queued */ + err = test_wait_fds(sk, last, is_writable, wait_for, TEST_TIMEOUT_SEC); + if (err < 0) + test_error("test_wait_fds(): %d", err); + + synchronize_threads(); /* 3: close listen socket */ + if (test_client_verify(sk[0], 100, quota / 100, TEST_TIMEOUT_SEC)) + test_fail("Failed to send data on connected socket"); + else + test_ok("Verified established tcp connection"); + + synchronize_threads(); /* 4: finishing up */ + err = _test_connect_socket(sk[last], this_ip_dest, port, -1); + if (err < 0) + test_error("failed to connect()"); + + synchronize_threads(); /* 5: closed active sk */ + err = test_wait_fds(sk, ARRAY_SIZE(sk), NULL, + wait_for, TEST_TIMEOUT_SEC); + if (err < 0) + test_error("select(): %d", err); + + for (i = 0; i < ARRAY_SIZE(sk); i++) { + socklen_t slen = sizeof(err); + + if (getsockopt(sk[i], SOL_SOCKET, SO_ERROR, &err, &slen)) + test_error("getsockopt()"); + if (is_writable[i] && err != ECONNRESET) { + test_fail("sk[%d] = %d, err = %d, connection wasn't reset", + i, sk[i], err); + } else { + test_ok("sk[%d] = %d%s", i, sk[i], + is_writable[i] ? ", connection was reset" : ""); + } + } + synchronize_threads(); /* 6: counters checks */ +} + +static void test_client_passive_rst(unsigned int port) +{ + struct tcp_ao_counters ao1, ao2; + struct tcp_ao_repair ao_img; + struct tcp_sock_state img; + sockaddr_af saddr; + int sk, err; + socklen_t slen = sizeof(err); + + sk = socket(test_family, SOCK_STREAM, IPPROTO_TCP); + if (sk < 0) + test_error("socket()"); + + if (test_add_key(sk, DEFAULT_TEST_PASSWORD, this_ip_dest, -1, 100, 100)) + test_error("setsockopt(TCP_AO_ADD_KEY)"); + + synchronize_threads(); /* 1: MKT added => connect() */ + if (test_connect_socket(sk, this_ip_dest, port) <= 0) + test_error("failed to connect()"); + + synchronize_threads(); /* 2: accepted => send data */ + if (test_client_verify(sk, 100, quota / 100, TEST_TIMEOUT_SEC)) + test_fail("Failed to send data on connected socket"); + else + test_ok("Verified established tcp connection"); + + synchronize_threads(); /* 3: chekpoint/restore the connection */ + test_enable_repair(sk); + test_sock_checkpoint(sk, &img, &saddr); + test_ao_checkpoint(sk, &ao_img); + test_kill_sk(sk); + + img.out.seq += quota; + + sk = socket(test_family, SOCK_STREAM, IPPROTO_TCP); + if (sk < 0) + test_error("socket()"); + + test_enable_repair(sk); + test_sock_restore(sk, &img, &saddr, this_ip_dest, port); + if (test_add_repaired_key(sk, DEFAULT_TEST_PASSWORD, 0, this_ip_dest, -1, 100, 100)) + test_error("setsockopt(TCP_AO_ADD_KEY)"); + test_ao_restore(sk, &ao_img); + + if (test_get_tcp_ao_counters(sk, &ao1)) + test_error("test_get_tcp_ao_counters()"); + + test_disable_repair(sk); + test_sock_state_free(&img); + + synchronize_threads(); /* 4: terminate server + send more on client */ + if (test_client_verify(sk, 100, quota / 100, 2 * TEST_TIMEOUT_SEC)) + test_ok("client connection broken post-seq-adjust"); + else + test_fail("client connection still works post-seq-adjust"); + + test_wait_for_exception(sk, TEST_TIMEOUT_SEC); + + if (getsockopt(sk, SOL_SOCKET, SO_ERROR, &err, &slen)) + test_error("getsockopt()"); + if (err != ECONNRESET && err != EPIPE) + test_fail("client connection was not reset: %d", err); + else + test_ok("client connection was reset"); + + if (test_get_tcp_ao_counters(sk, &ao2)) + test_error("test_get_tcp_ao_counters()"); + + synchronize_threads(); /* 5: verified => closed */ + close(sk); + test_tcp_ao_counters_cmp("client passive RST", &ao1, &ao2, TEST_CNT_GOOD); +} + +static void *client_fn(void *arg) +{ + struct netstat *ns_before, *ns_after; + unsigned int port = test_server_port; + + ns_before = netstat_read(); + + test_client_active_rst(port++); + test_client_passive_rst(port++); + + ns_after = netstat_read(); + netstats_check(ns_before, ns_after, "client"); + netstat_free(ns_after); + netstat_free(ns_before); + + synchronize_threads(); /* exit */ + return NULL; +} + +int main(int argc, char *argv[]) +{ + test_init(15, server_fn, client_fn); + return 0; +} diff --git a/tools/testing/selftests/net/tcp_ao/self-connect.c b/tools/testing/selftests/net/tcp_ao/self-connect.c new file mode 100644 index 00000000000000..e154d9e198a993 --- /dev/null +++ b/tools/testing/selftests/net/tcp_ao/self-connect.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Author: Dmitry Safonov */ +#include +#include "aolib.h" + +static union tcp_addr local_addr; + +static void __setup_lo_intf(const char *lo_intf, + const char *addr_str, uint8_t prefix) +{ + if (inet_pton(TEST_FAMILY, addr_str, &local_addr) != 1) + test_error("Can't convert local ip address"); + + if (ip_addr_add(lo_intf, TEST_FAMILY, local_addr, prefix)) + test_error("Failed to add %s ip address", lo_intf); + + if (link_set_up(lo_intf)) + test_error("Failed to bring %s up", lo_intf); +} + +static void setup_lo_intf(const char *lo_intf) +{ +#ifdef IPV6_TEST + __setup_lo_intf(lo_intf, "::1", 128); +#else + __setup_lo_intf(lo_intf, "127.0.0.1", 8); +#endif +} + +static void tcp_self_connect(const char *tst, unsigned int port, + bool different_keyids, bool check_restore) +{ + uint64_t before_challenge_ack, after_challenge_ack; + uint64_t before_syn_challenge, after_syn_challenge; + struct tcp_ao_counters before_ao, after_ao; + uint64_t before_aogood, after_aogood; + struct netstat *ns_before, *ns_after; + const size_t nr_packets = 20; + struct tcp_ao_repair ao_img; + struct tcp_sock_state img; + sockaddr_af addr; + int sk; + + tcp_addr_to_sockaddr_in(&addr, &local_addr, htons(port)); + + sk = socket(test_family, SOCK_STREAM, IPPROTO_TCP); + if (sk < 0) + test_error("socket()"); + + if (different_keyids) { + if (test_add_key(sk, DEFAULT_TEST_PASSWORD, local_addr, -1, 5, 7)) + test_error("setsockopt(TCP_AO_ADD_KEY)"); + if (test_add_key(sk, DEFAULT_TEST_PASSWORD, local_addr, -1, 7, 5)) + test_error("setsockopt(TCP_AO_ADD_KEY)"); + } else { + if (test_add_key(sk, DEFAULT_TEST_PASSWORD, local_addr, -1, 100, 100)) + test_error("setsockopt(TCP_AO_ADD_KEY)"); + } + + if (bind(sk, (struct sockaddr *)&addr, sizeof(addr)) < 0) + test_error("bind()"); + + ns_before = netstat_read(); + before_aogood = netstat_get(ns_before, "TCPAOGood", NULL); + before_challenge_ack = netstat_get(ns_before, "TCPChallengeACK", NULL); + before_syn_challenge = netstat_get(ns_before, "TCPSYNChallenge", NULL); + if (test_get_tcp_ao_counters(sk, &before_ao)) + test_error("test_get_tcp_ao_counters()"); + + if (__test_connect_socket(sk, "lo", (struct sockaddr *)&addr, + sizeof(addr), TEST_TIMEOUT_SEC) < 0) { + ns_after = netstat_read(); + netstat_print_diff(ns_before, ns_after); + test_error("failed to connect()"); + } + + if (test_client_verify(sk, 100, nr_packets, TEST_TIMEOUT_SEC)) { + test_fail("%s: tcp connection verify failed", tst); + close(sk); + return; + } + + ns_after = netstat_read(); + after_aogood = netstat_get(ns_after, "TCPAOGood", NULL); + after_challenge_ack = netstat_get(ns_after, "TCPChallengeACK", NULL); + after_syn_challenge = netstat_get(ns_after, "TCPSYNChallenge", NULL); + if (test_get_tcp_ao_counters(sk, &after_ao)) + test_error("test_get_tcp_ao_counters()"); + if (!check_restore) { + /* to debug: netstat_print_diff(ns_before, ns_after); */ + netstat_free(ns_before); + } + netstat_free(ns_after); + + if (after_aogood <= before_aogood) { + test_fail("%s: TCPAOGood counter mismatch: %zu <= %zu", + tst, after_aogood, before_aogood); + close(sk); + return; + } + if (after_challenge_ack <= before_challenge_ack || + after_syn_challenge <= before_syn_challenge) { + /* + * It's also meant to test simultaneous open, so check + * these counters as well. + */ + test_fail("%s: Didn't challenge SYN or ACK: %zu <= %zu OR %zu <= %zu", + tst, after_challenge_ack, before_challenge_ack, + after_syn_challenge, before_syn_challenge); + close(sk); + return; + } + + if (test_tcp_ao_counters_cmp(tst, &before_ao, &after_ao, TEST_CNT_GOOD)) { + close(sk); + return; + } + + if (!check_restore) { + test_ok("%s: connect TCPAOGood %" PRIu64 " => %" PRIu64, + tst, before_aogood, after_aogood); + close(sk); + return; + } + + test_enable_repair(sk); + test_sock_checkpoint(sk, &img, &addr); +#ifdef IPV6_TEST + addr.sin6_port = htons(port + 1); +#else + addr.sin_port = htons(port + 1); +#endif + test_ao_checkpoint(sk, &ao_img); + test_kill_sk(sk); + + sk = socket(test_family, SOCK_STREAM, IPPROTO_TCP); + if (sk < 0) + test_error("socket()"); + + test_enable_repair(sk); + __test_sock_restore(sk, "lo", &img, &addr, &addr, sizeof(addr)); + if (different_keyids) { + if (test_add_repaired_key(sk, DEFAULT_TEST_PASSWORD, 0, + local_addr, -1, 7, 5)) + test_error("setsockopt(TCP_AO_ADD_KEY)"); + if (test_add_repaired_key(sk, DEFAULT_TEST_PASSWORD, 0, + local_addr, -1, 5, 7)) + test_error("setsockopt(TCP_AO_ADD_KEY)"); + } else { + if (test_add_repaired_key(sk, DEFAULT_TEST_PASSWORD, 0, + local_addr, -1, 100, 100)) + test_error("setsockopt(TCP_AO_ADD_KEY)"); + } + test_ao_restore(sk, &ao_img); + test_disable_repair(sk); + test_sock_state_free(&img); + if (test_client_verify(sk, 100, nr_packets, TEST_TIMEOUT_SEC)) { + test_fail("%s: tcp connection verify failed", tst); + close(sk); + return; + } + ns_after = netstat_read(); + after_aogood = netstat_get(ns_after, "TCPAOGood", NULL); + /* to debug: netstat_print_diff(ns_before, ns_after); */ + netstat_free(ns_before); + netstat_free(ns_after); + close(sk); + if (after_aogood <= before_aogood) { + test_fail("%s: TCPAOGood counter mismatch: %zu <= %zu", + tst, after_aogood, before_aogood); + return; + } + test_ok("%s: connect TCPAOGood %" PRIu64 " => %" PRIu64, + tst, before_aogood, after_aogood); +} + +static void *client_fn(void *arg) +{ + unsigned int port = test_server_port; + + setup_lo_intf("lo"); + + tcp_self_connect("self-connect(same keyids)", port++, false, false); + tcp_self_connect("self-connect(different keyids)", port++, true, false); + tcp_self_connect("self-connect(restore)", port, false, true); + port += 2; + tcp_self_connect("self-connect(restore, different keyids)", port, true, true); + port += 2; + + return NULL; +} + +int main(int argc, char *argv[]) +{ + test_init(4, client_fn, NULL); + return 0; +} diff --git a/tools/testing/selftests/net/tcp_ao/seq-ext.c b/tools/testing/selftests/net/tcp_ao/seq-ext.c new file mode 100644 index 00000000000000..ad4e77d6823ee3 --- /dev/null +++ b/tools/testing/selftests/net/tcp_ao/seq-ext.c @@ -0,0 +1,245 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Check that after SEQ number wrap-around: + * 1. SEQ-extension has upper bytes set + * 2. TCP conneciton is alive and no TCPAOBad segments + * In order to test (2), the test doesn't just adjust seq number for a queue + * on a connected socket, but migrates it to another sk+port number, so + * that there won't be any delayed packets that will fail to verify + * with the new SEQ numbers. + */ +#include +#include "aolib.h" + +const unsigned int nr_packets = 1000; +const unsigned int msg_len = 1000; +const unsigned int quota = nr_packets * msg_len; +unsigned int client_new_port; + +/* Move them closer to roll-over */ +static void test_adjust_seqs(struct tcp_sock_state *img, + struct tcp_ao_repair *ao_img, + bool server) +{ + uint32_t new_seq1, new_seq2; + + /* make them roll-over during quota, but on different segments */ + if (server) { + new_seq1 = ((uint32_t)-1) - msg_len; + new_seq2 = ((uint32_t)-1) - (quota - 2 * msg_len); + } else { + new_seq1 = ((uint32_t)-1) - (quota - 2 * msg_len); + new_seq2 = ((uint32_t)-1) - msg_len; + } + + img->in.seq = new_seq1; + img->trw.snd_wl1 = img->in.seq - msg_len; + img->out.seq = new_seq2; + img->trw.rcv_wup = img->in.seq; +} + +static int test_sk_restore(struct tcp_sock_state *img, + struct tcp_ao_repair *ao_img, sockaddr_af *saddr, + const union tcp_addr daddr, unsigned int dport, + struct tcp_ao_counters *cnt) +{ + int sk; + + sk = socket(test_family, SOCK_STREAM, IPPROTO_TCP); + if (sk < 0) + test_error("socket()"); + + test_enable_repair(sk); + test_sock_restore(sk, img, saddr, daddr, dport); + if (test_add_repaired_key(sk, DEFAULT_TEST_PASSWORD, 0, daddr, -1, 100, 100)) + test_error("setsockopt(TCP_AO_ADD_KEY)"); + test_ao_restore(sk, ao_img); + + if (test_get_tcp_ao_counters(sk, cnt)) + test_error("test_get_tcp_ao_counters()"); + + test_disable_repair(sk); + test_sock_state_free(img); + return sk; +} + +static void *server_fn(void *arg) +{ + uint64_t before_good, after_good, after_bad; + struct tcp_ao_counters ao1, ao2; + struct tcp_sock_state img; + struct tcp_ao_repair ao_img; + sockaddr_af saddr; + ssize_t bytes; + int sk, lsk; + + lsk = test_listen_socket(this_ip_addr, test_server_port, 1); + + if (test_add_key(lsk, DEFAULT_TEST_PASSWORD, this_ip_dest, -1, 100, 100)) + test_error("setsockopt(TCP_AO_ADD_KEY)"); + + synchronize_threads(); /* 1: MKT added => connect() */ + + if (test_wait_fd(lsk, TEST_TIMEOUT_SEC, 0)) + test_error("test_wait_fd()"); + + sk = accept(lsk, NULL, NULL); + if (sk < 0) + test_error("accept()"); + + synchronize_threads(); /* 2: accepted => send data */ + close(lsk); + + bytes = test_server_run(sk, quota, TEST_TIMEOUT_SEC); + if (bytes != quota) { + if (bytes > 0) + test_fail("server served: %zd", bytes); + else + test_fail("server returned: %zd", bytes); + goto out; + } + + before_good = netstat_get_one("TCPAOGood", NULL); + + synchronize_threads(); /* 3: restore the connection on another port */ + + test_enable_repair(sk); + test_sock_checkpoint(sk, &img, &saddr); + test_ao_checkpoint(sk, &ao_img); + test_kill_sk(sk); +#ifdef IPV6_TEST + saddr.sin6_port = htons(ntohs(saddr.sin6_port) + 1); +#else + saddr.sin_port = htons(ntohs(saddr.sin_port) + 1); +#endif + test_adjust_seqs(&img, &ao_img, true); + synchronize_threads(); /* 4: dump finished */ + sk = test_sk_restore(&img, &ao_img, &saddr, this_ip_dest, + client_new_port, &ao1); + + synchronize_threads(); /* 5: verify counters during SEQ-number rollover */ + bytes = test_server_run(sk, quota, TEST_TIMEOUT_SEC); + if (bytes != quota) { + if (bytes > 0) + test_fail("server served: %zd", bytes); + else + test_fail("server returned: %zd", bytes); + } else { + test_ok("server alive"); + } + + if (test_get_tcp_ao_counters(sk, &ao2)) + test_error("test_get_tcp_ao_counters()"); + after_good = netstat_get_one("TCPAOGood", NULL); + + test_tcp_ao_counters_cmp(NULL, &ao1, &ao2, TEST_CNT_GOOD); + + if (after_good <= before_good) { + test_fail("TCPAOGood counter did not increase: %zu <= %zu", + after_good, before_good); + } else { + test_ok("TCPAOGood counter increased %zu => %zu", + before_good, after_good); + } + after_bad = netstat_get_one("TCPAOBad", NULL); + if (after_bad) + test_fail("TCPAOBad counter is non-zero: %zu", after_bad); + else + test_ok("TCPAOBad counter didn't increase"); + test_enable_repair(sk); + test_ao_checkpoint(sk, &ao_img); + if (ao_img.snd_sne && ao_img.rcv_sne) { + test_ok("SEQ extension incremented: %u/%u", + ao_img.snd_sne, ao_img.rcv_sne); + } else { + test_fail("SEQ extension was not incremented: %u/%u", + ao_img.snd_sne, ao_img.rcv_sne); + } + + synchronize_threads(); /* 6: verified => closed */ +out: + close(sk); + return NULL; +} + +static void *client_fn(void *arg) +{ + uint64_t before_good, after_good, after_bad; + struct tcp_ao_counters ao1, ao2; + struct tcp_sock_state img; + struct tcp_ao_repair ao_img; + sockaddr_af saddr; + int sk; + + sk = socket(test_family, SOCK_STREAM, IPPROTO_TCP); + if (sk < 0) + test_error("socket()"); + + if (test_add_key(sk, DEFAULT_TEST_PASSWORD, this_ip_dest, -1, 100, 100)) + test_error("setsockopt(TCP_AO_ADD_KEY)"); + + synchronize_threads(); /* 1: MKT added => connect() */ + if (test_connect_socket(sk, this_ip_dest, test_server_port) <= 0) + test_error("failed to connect()"); + + synchronize_threads(); /* 2: accepted => send data */ + if (test_client_verify(sk, msg_len, nr_packets, TEST_TIMEOUT_SEC)) { + test_fail("pre-migrate verify failed"); + return NULL; + } + + before_good = netstat_get_one("TCPAOGood", NULL); + + synchronize_threads(); /* 3: restore the connection on another port */ + test_enable_repair(sk); + test_sock_checkpoint(sk, &img, &saddr); + test_ao_checkpoint(sk, &ao_img); + test_kill_sk(sk); +#ifdef IPV6_TEST + client_new_port = ntohs(saddr.sin6_port) + 1; + saddr.sin6_port = htons(ntohs(saddr.sin6_port) + 1); +#else + client_new_port = ntohs(saddr.sin_port) + 1; + saddr.sin_port = htons(ntohs(saddr.sin_port) + 1); +#endif + test_adjust_seqs(&img, &ao_img, false); + synchronize_threads(); /* 4: dump finished */ + sk = test_sk_restore(&img, &ao_img, &saddr, this_ip_dest, + test_server_port + 1, &ao1); + + synchronize_threads(); /* 5: verify counters during SEQ-number rollover */ + if (test_client_verify(sk, msg_len, nr_packets, TEST_TIMEOUT_SEC)) + test_fail("post-migrate verify failed"); + else + test_ok("post-migrate connection alive"); + + if (test_get_tcp_ao_counters(sk, &ao2)) + test_error("test_get_tcp_ao_counters()"); + after_good = netstat_get_one("TCPAOGood", NULL); + + test_tcp_ao_counters_cmp(NULL, &ao1, &ao2, TEST_CNT_GOOD); + + if (after_good <= before_good) { + test_fail("TCPAOGood counter did not increase: %zu <= %zu", + after_good, before_good); + } else { + test_ok("TCPAOGood counter increased %zu => %zu", + before_good, after_good); + } + after_bad = netstat_get_one("TCPAOBad", NULL); + if (after_bad) + test_fail("TCPAOBad counter is non-zero: %zu", after_bad); + else + test_ok("TCPAOBad counter didn't increase"); + + synchronize_threads(); /* 6: verified => closed */ + close(sk); + + synchronize_threads(); /* don't race to exit: let server exit() */ + return NULL; +} + +int main(int argc, char *argv[]) +{ + test_init(7, server_fn, client_fn); + return 0; +} diff --git a/tools/testing/selftests/net/tcp_ao/setsockopt-closed.c b/tools/testing/selftests/net/tcp_ao/setsockopt-closed.c new file mode 100644 index 00000000000000..7e4601b3f6a309 --- /dev/null +++ b/tools/testing/selftests/net/tcp_ao/setsockopt-closed.c @@ -0,0 +1,835 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Author: Dmitry Safonov */ +#include +#include "../../../../include/linux/kernel.h" +#include "aolib.h" + +static union tcp_addr tcp_md5_client; + +static int test_port = 7788; +static void make_listen(int sk) +{ + sockaddr_af addr; + + tcp_addr_to_sockaddr_in(&addr, &this_ip_addr, htons(test_port++)); + if (bind(sk, (struct sockaddr *)&addr, sizeof(addr)) < 0) + test_error("bind()"); + if (listen(sk, 1)) + test_error("listen()"); +} + +static void test_vefify_ao_info(int sk, struct tcp_ao_info_opt *info, + const char *tst) +{ + struct tcp_ao_info_opt tmp; + socklen_t len = sizeof(tmp); + + if (getsockopt(sk, IPPROTO_TCP, TCP_AO_INFO, &tmp, &len)) + test_error("getsockopt(TCP_AO_INFO) failed"); + +#define __cmp_ao(member) \ +do { \ + if (info->member != tmp.member) { \ + test_fail("%s: getsockopt(): " __stringify(member) " %zu != %zu", \ + tst, (size_t)info->member, (size_t)tmp.member); \ + return; \ + } \ +} while(0) + if (info->set_current) + __cmp_ao(current_key); + if (info->set_rnext) + __cmp_ao(rnext); + if (info->set_counters) { + __cmp_ao(pkt_good); + __cmp_ao(pkt_bad); + __cmp_ao(pkt_key_not_found); + __cmp_ao(pkt_ao_required); + __cmp_ao(pkt_dropped_icmp); + } + __cmp_ao(ao_required); + __cmp_ao(accept_icmps); + + test_ok("AO info get: %s", tst); +#undef __cmp_ao +} + +static void __setsockopt_checked(int sk, int optname, bool get, + void *optval, socklen_t *len, + int err, const char *tst, const char *tst2) +{ + int ret; + + if (!tst) + tst = ""; + if (!tst2) + tst2 = ""; + + errno = 0; + if (get) + ret = getsockopt(sk, IPPROTO_TCP, optname, optval, len); + else + ret = setsockopt(sk, IPPROTO_TCP, optname, optval, *len); + if (ret == -1) { + if (errno == err) + test_ok("%s%s", tst ?: "", tst2 ?: ""); + else + test_fail("%s%s: %setsockopt() failed", + tst, tst2, get ? "g" : "s"); + close(sk); + return; + } + + if (err) { + test_fail("%s%s: %setsockopt() was expected to fail with %d", + tst, tst2, get ? "g" : "s", err); + } else { + test_ok("%s%s", tst ?: "", tst2 ?: ""); + if (optname == TCP_AO_ADD_KEY) { + test_verify_socket_key(sk, optval); + } else if (optname == TCP_AO_INFO && !get) { + test_vefify_ao_info(sk, optval, tst2); + } else if (optname == TCP_AO_GET_KEYS) { + if (*len != sizeof(struct tcp_ao_getsockopt)) + test_fail("%s%s: get keys returned wrong tcp_ao_getsockopt size", + tst, tst2); + } + } + close(sk); +} + +static void setsockopt_checked(int sk, int optname, void *optval, + int err, const char *tst) +{ + const char *cmd = NULL; + socklen_t len; + + switch (optname) { + case TCP_AO_ADD_KEY: + cmd = "key add: "; + len = sizeof(struct tcp_ao_add); + break; + case TCP_AO_DEL_KEY: + cmd = "key del: "; + len = sizeof(struct tcp_ao_del); + break; + case TCP_AO_INFO: + cmd = "AO info set: "; + len = sizeof(struct tcp_ao_info_opt); + break; + default: + break; + }; + + __setsockopt_checked(sk, optname, false, optval, &len, err, cmd, tst); +} + +static int prepare_defs(int cmd, void *optval) +{ + int sk = socket(test_family, SOCK_STREAM, IPPROTO_TCP); + + if (sk < 0) + test_error("socket()"); + + switch (cmd) { + case TCP_AO_ADD_KEY: { + struct tcp_ao_add *add = optval; + + if (test_prepare_def_key(add, DEFAULT_TEST_PASSWORD, 0, this_ip_dest, + -1, 0, 100, 100)) + test_error("prepare default tcp_ao_add"); + break; + } + case TCP_AO_DEL_KEY: { + struct tcp_ao_del *del = optval; + + if (test_add_key(sk, DEFAULT_TEST_PASSWORD, this_ip_dest, + DEFAULT_TEST_PREFIX, 100, 100)) + test_error("add default key"); + memset(del, 0, sizeof(struct tcp_ao_del)); + del->sndid = 100; + del->rcvid = 100; + del->prefix = DEFAULT_TEST_PREFIX; + tcp_addr_to_sockaddr_in(&del->addr, &this_ip_dest, 0); + break; + } + case TCP_AO_INFO: { + struct tcp_ao_info_opt *info = optval; + + if (test_add_key(sk, DEFAULT_TEST_PASSWORD, this_ip_dest, + DEFAULT_TEST_PREFIX, 100, 100)) + test_error("add default key"); + memset(info, 0, sizeof(struct tcp_ao_info_opt)); + break; + } + case TCP_AO_GET_KEYS: { + struct tcp_ao_getsockopt *get = optval; + + if (test_add_key(sk, DEFAULT_TEST_PASSWORD, this_ip_dest, + DEFAULT_TEST_PREFIX, 100, 100)) + test_error("add default key"); + memset(get, 0, sizeof(struct tcp_ao_getsockopt)); + get->nkeys = 1; + get->get_all = 1; + break; + } + default: + test_error("unknown cmd"); + } + + return sk; +} + +static void test_extend(int cmd, bool get, const char *tst, socklen_t under_size) +{ + struct { + union { + struct tcp_ao_add add; + struct tcp_ao_del del; + struct tcp_ao_getsockopt get; + struct tcp_ao_info_opt info; + }; + char *extend[100]; + } tmp_opt; + socklen_t extended_size = sizeof(tmp_opt); + int sk; + + memset(&tmp_opt, 0, sizeof(tmp_opt)); + sk = prepare_defs(cmd, &tmp_opt); + __setsockopt_checked(sk, cmd, get, &tmp_opt, &under_size, + EINVAL, tst, ": minimum size"); + + memset(&tmp_opt, 0, sizeof(tmp_opt)); + sk = prepare_defs(cmd, &tmp_opt); + __setsockopt_checked(sk, cmd, get, &tmp_opt, &extended_size, + 0, tst, ": extended size"); + + memset(&tmp_opt, 0, sizeof(tmp_opt)); + sk = prepare_defs(cmd, &tmp_opt); + __setsockopt_checked(sk, cmd, get, NULL, &extended_size, + EFAULT, tst, ": null optval"); + + if (get) { + memset(&tmp_opt, 0, sizeof(tmp_opt)); + sk = prepare_defs(cmd, &tmp_opt); + __setsockopt_checked(sk, cmd, get, &tmp_opt, NULL, + EFAULT, tst, ": null optlen"); + } +} + +static void extend_tests(void) +{ + test_extend(TCP_AO_ADD_KEY, false, "AO add", + offsetof(struct tcp_ao_add, key)); + test_extend(TCP_AO_DEL_KEY, false, "AO del", + offsetof(struct tcp_ao_del, keyflags)); + test_extend(TCP_AO_INFO, false, "AO set info", + offsetof(struct tcp_ao_info_opt, pkt_dropped_icmp)); + test_extend(TCP_AO_INFO, true, "AO get info", -1); + test_extend(TCP_AO_GET_KEYS, true, "AO get keys", -1); +} + +static void test_optmem_limit(void) +{ + size_t i, keys_limit, current_optmem = test_get_optmem(); + struct tcp_ao_add ao; + union tcp_addr net = {}; + int sk; + + if (inet_pton(TEST_FAMILY, TEST_NETWORK, &net) != 1) + test_error("Can't convert ip address %s", TEST_NETWORK); + + sk = prepare_defs(TCP_AO_ADD_KEY, &ao); + keys_limit = current_optmem / KERNEL_TCP_AO_KEY_SZ_ROUND_UP; + for (i = 0;; i++) { + union tcp_addr key_peer; + int err; + + key_peer = gen_tcp_addr(net, i + 1); + tcp_addr_to_sockaddr_in(&ao.addr, &key_peer, 0); + err = setsockopt(sk, IPPROTO_TCP, TCP_AO_ADD_KEY, + &ao, sizeof(ao)); + if (!err) { + /* + * TCP_AO_ADD_KEY should be the same order as the real + * sizeof(struct tcp_ao_key) in kernel. + */ + if (i <= keys_limit * 10) + continue; + test_fail("optmem limit test failed: added %zu key", i); + break; + } + if (i < keys_limit) { + test_fail("optmem limit test failed: couldn't add %zu key", i); + break; + } + test_ok("optmem limit was hit on adding %zu key", i); + break; + } + close(sk); +} + +static void test_einval_add_key(void) +{ + struct tcp_ao_add ao; + int sk; + + sk = prepare_defs(TCP_AO_ADD_KEY, &ao); + ao.keylen = TCP_AO_MAXKEYLEN + 1; + setsockopt_checked(sk, TCP_AO_ADD_KEY, &ao, EINVAL, "too big keylen"); + + sk = prepare_defs(TCP_AO_ADD_KEY, &ao); + ao.reserved = 1; + setsockopt_checked(sk, TCP_AO_ADD_KEY, &ao, EINVAL, "using reserved padding"); + + sk = prepare_defs(TCP_AO_ADD_KEY, &ao); + ao.reserved2 = 1; + setsockopt_checked(sk, TCP_AO_ADD_KEY, &ao, EINVAL, "using reserved2 padding"); + + /* tcp_ao_verify_ipv{4,6}() checks */ + sk = prepare_defs(TCP_AO_ADD_KEY, &ao); + ao.addr.ss_family = AF_UNIX; + memcpy(&ao.addr, &SOCKADDR_ANY, sizeof(SOCKADDR_ANY)); + setsockopt_checked(sk, TCP_AO_ADD_KEY, &ao, EINVAL, "wrong address family"); + + sk = prepare_defs(TCP_AO_ADD_KEY, &ao); + tcp_addr_to_sockaddr_in(&ao.addr, &this_ip_dest, 1234); + setsockopt_checked(sk, TCP_AO_ADD_KEY, &ao, EINVAL, "port (unsupported)"); + + sk = prepare_defs(TCP_AO_ADD_KEY, &ao); + ao.prefix = 0; + setsockopt_checked(sk, TCP_AO_ADD_KEY, &ao, EINVAL, "no prefix, addr"); + + sk = prepare_defs(TCP_AO_ADD_KEY, &ao); + ao.prefix = 0; + memcpy(&ao.addr, &SOCKADDR_ANY, sizeof(SOCKADDR_ANY)); + setsockopt_checked(sk, TCP_AO_ADD_KEY, &ao, 0, "no prefix, any addr"); + + sk = prepare_defs(TCP_AO_ADD_KEY, &ao); + ao.prefix = 32; + memcpy(&ao.addr, &SOCKADDR_ANY, sizeof(SOCKADDR_ANY)); + setsockopt_checked(sk, TCP_AO_ADD_KEY, &ao, EINVAL, "prefix, any addr"); + + sk = prepare_defs(TCP_AO_ADD_KEY, &ao); + ao.prefix = 129; + setsockopt_checked(sk, TCP_AO_ADD_KEY, &ao, EINVAL, "too big prefix"); + + sk = prepare_defs(TCP_AO_ADD_KEY, &ao); + ao.prefix = 2; + setsockopt_checked(sk, TCP_AO_ADD_KEY, &ao, EINVAL, "too short prefix"); + + sk = prepare_defs(TCP_AO_ADD_KEY, &ao); + ao.keyflags = (uint8_t)(-1); + setsockopt_checked(sk, TCP_AO_ADD_KEY, &ao, EINVAL, "bad key flags"); + + sk = prepare_defs(TCP_AO_ADD_KEY, &ao); + make_listen(sk); + ao.set_current = 1; + setsockopt_checked(sk, TCP_AO_ADD_KEY, &ao, EINVAL, "add current key on a listen socket"); + + sk = prepare_defs(TCP_AO_ADD_KEY, &ao); + make_listen(sk); + ao.set_rnext = 1; + setsockopt_checked(sk, TCP_AO_ADD_KEY, &ao, EINVAL, "add rnext key on a listen socket"); + + sk = prepare_defs(TCP_AO_ADD_KEY, &ao); + make_listen(sk); + ao.set_current = 1; + ao.set_rnext = 1; + setsockopt_checked(sk, TCP_AO_ADD_KEY, &ao, EINVAL, "add current+rnext key on a listen socket"); + + sk = prepare_defs(TCP_AO_ADD_KEY, &ao); + ao.set_current = 1; + setsockopt_checked(sk, TCP_AO_ADD_KEY, &ao, 0, "add key and set as current"); + + sk = prepare_defs(TCP_AO_ADD_KEY, &ao); + ao.set_rnext = 1; + setsockopt_checked(sk, TCP_AO_ADD_KEY, &ao, 0, "add key and set as rnext"); + + sk = prepare_defs(TCP_AO_ADD_KEY, &ao); + ao.set_current = 1; + ao.set_rnext = 1; + setsockopt_checked(sk, TCP_AO_ADD_KEY, &ao, 0, "add key and set as current+rnext"); + + sk = prepare_defs(TCP_AO_ADD_KEY, &ao); + ao.ifindex = 42; + setsockopt_checked(sk, TCP_AO_ADD_KEY, &ao, EINVAL, + "ifindex without TCP_AO_KEYF_IFNINDEX"); + + sk = prepare_defs(TCP_AO_ADD_KEY, &ao); + ao.keyflags |= TCP_AO_KEYF_IFINDEX; + ao.ifindex = 42; + setsockopt_checked(sk, TCP_AO_ADD_KEY, &ao, EINVAL, "non-existent VRF"); + /* + * tcp_md5_do_lookup{,_any_l3index}() are checked in unsigned-md5 + * see client_vrf_tests(). + */ + + test_optmem_limit(); + + /* tcp_ao_parse_crypto() */ + sk = prepare_defs(TCP_AO_ADD_KEY, &ao); + ao.maclen = 100; + setsockopt_checked(sk, TCP_AO_ADD_KEY, &ao, EMSGSIZE, "maclen bigger than TCP hdr"); + + sk = prepare_defs(TCP_AO_ADD_KEY, &ao); + strcpy(ao.alg_name, "imaginary hash algo"); + setsockopt_checked(sk, TCP_AO_ADD_KEY, &ao, ENOENT, "bad algo"); +} + +static void test_einval_del_key(void) +{ + struct tcp_ao_del del; + int sk; + + sk = prepare_defs(TCP_AO_DEL_KEY, &del); + del.reserved = 1; + setsockopt_checked(sk, TCP_AO_DEL_KEY, &del, EINVAL, "using reserved padding"); + + sk = prepare_defs(TCP_AO_DEL_KEY, &del); + del.reserved2 = 1; + setsockopt_checked(sk, TCP_AO_DEL_KEY, &del, EINVAL, "using reserved2 padding"); + + sk = prepare_defs(TCP_AO_DEL_KEY, &del); + make_listen(sk); + if (test_add_key(sk, DEFAULT_TEST_PASSWORD, this_ip_dest, DEFAULT_TEST_PREFIX, 0, 0)) + test_error("add key"); + del.set_current = 1; + setsockopt_checked(sk, TCP_AO_DEL_KEY, &del, EINVAL, "del and set current key on a listen socket"); + + sk = prepare_defs(TCP_AO_DEL_KEY, &del); + make_listen(sk); + if (test_add_key(sk, DEFAULT_TEST_PASSWORD, this_ip_dest, DEFAULT_TEST_PREFIX, 0, 0)) + test_error("add key"); + del.set_rnext = 1; + setsockopt_checked(sk, TCP_AO_DEL_KEY, &del, EINVAL, "del and set rnext key on a listen socket"); + + sk = prepare_defs(TCP_AO_DEL_KEY, &del); + make_listen(sk); + if (test_add_key(sk, DEFAULT_TEST_PASSWORD, this_ip_dest, DEFAULT_TEST_PREFIX, 0, 0)) + test_error("add key"); + del.set_current = 1; + del.set_rnext = 1; + setsockopt_checked(sk, TCP_AO_DEL_KEY, &del, EINVAL, "del and set current+rnext key on a listen socket"); + + sk = prepare_defs(TCP_AO_DEL_KEY, &del); + del.keyflags = (uint8_t)(-1); + setsockopt_checked(sk, TCP_AO_DEL_KEY, &del, EINVAL, "bad key flags"); + + sk = prepare_defs(TCP_AO_DEL_KEY, &del); + del.ifindex = 42; + setsockopt_checked(sk, TCP_AO_DEL_KEY, &del, EINVAL, + "ifindex without TCP_AO_KEYF_IFNINDEX"); + + sk = prepare_defs(TCP_AO_DEL_KEY, &del); + del.keyflags |= TCP_AO_KEYF_IFINDEX; + del.ifindex = 42; + setsockopt_checked(sk, TCP_AO_DEL_KEY, &del, ENOENT, "non-existent VRF"); + + sk = prepare_defs(TCP_AO_DEL_KEY, &del); + del.set_current = 1; + setsockopt_checked(sk, TCP_AO_DEL_KEY, &del, ENOENT, "set non-exising current key"); + + sk = prepare_defs(TCP_AO_DEL_KEY, &del); + del.set_rnext = 1; + setsockopt_checked(sk, TCP_AO_DEL_KEY, &del, ENOENT, "set non-existing rnext key"); + + sk = prepare_defs(TCP_AO_DEL_KEY, &del); + del.set_current = 1; + del.set_rnext = 1; + setsockopt_checked(sk, TCP_AO_DEL_KEY, &del, ENOENT, "set non-existing current+rnext key"); + + sk = prepare_defs(TCP_AO_DEL_KEY, &del); + if (test_add_key(sk, DEFAULT_TEST_PASSWORD, this_ip_dest, DEFAULT_TEST_PREFIX, 0, 0)) + test_error("add key"); + del.set_current = 1; + setsockopt_checked(sk, TCP_AO_DEL_KEY, &del, 0, "set current key"); + + sk = prepare_defs(TCP_AO_DEL_KEY, &del); + if (test_add_key(sk, DEFAULT_TEST_PASSWORD, this_ip_dest, DEFAULT_TEST_PREFIX, 0, 0)) + test_error("add key"); + del.set_rnext = 1; + setsockopt_checked(sk, TCP_AO_DEL_KEY, &del, 0, "set rnext key"); + + sk = prepare_defs(TCP_AO_DEL_KEY, &del); + if (test_add_key(sk, DEFAULT_TEST_PASSWORD, this_ip_dest, DEFAULT_TEST_PREFIX, 0, 0)) + test_error("add key"); + del.set_current = 1; + del.set_rnext = 1; + setsockopt_checked(sk, TCP_AO_DEL_KEY, &del, 0, "set current+rnext key"); + + sk = prepare_defs(TCP_AO_DEL_KEY, &del); + del.set_current = 1; + del.current_key = 100; + setsockopt_checked(sk, TCP_AO_DEL_KEY, &del, ENOENT, "set as current key to be removed"); + + sk = prepare_defs(TCP_AO_DEL_KEY, &del); + del.set_rnext = 1; + del.rnext = 100; + setsockopt_checked(sk, TCP_AO_DEL_KEY, &del, ENOENT, "set as rnext key to be removed"); + + sk = prepare_defs(TCP_AO_DEL_KEY, &del); + del.set_current = 1; + del.current_key = 100; + del.set_rnext = 1; + del.rnext = 100; + setsockopt_checked(sk, TCP_AO_DEL_KEY, &del, ENOENT, "set as current+rnext key to be removed"); + + sk = prepare_defs(TCP_AO_DEL_KEY, &del); + del.del_async = 1; + setsockopt_checked(sk, TCP_AO_DEL_KEY, &del, EINVAL, "async on non-listen"); + + sk = prepare_defs(TCP_AO_DEL_KEY, &del); + del.sndid = 101; + setsockopt_checked(sk, TCP_AO_DEL_KEY, &del, ENOENT, "non-existing sndid"); + + sk = prepare_defs(TCP_AO_DEL_KEY, &del); + del.rcvid = 101; + setsockopt_checked(sk, TCP_AO_DEL_KEY, &del, ENOENT, "non-existing rcvid"); + + sk = prepare_defs(TCP_AO_DEL_KEY, &del); + tcp_addr_to_sockaddr_in(&del.addr, &this_ip_addr, 0); + setsockopt_checked(sk, TCP_AO_DEL_KEY, &del, ENOENT, "incorrect addr"); + + sk = prepare_defs(TCP_AO_DEL_KEY, &del); + setsockopt_checked(sk, TCP_AO_DEL_KEY, &del, 0, "correct key delete"); +} + +static void test_einval_ao_info(void) +{ + struct tcp_ao_info_opt info; + int sk; + + sk = prepare_defs(TCP_AO_INFO, &info); + make_listen(sk); + info.set_current = 1; + setsockopt_checked(sk, TCP_AO_INFO, &info, EINVAL, "set current key on a listen socket"); + + sk = prepare_defs(TCP_AO_INFO, &info); + make_listen(sk); + info.set_rnext = 1; + setsockopt_checked(sk, TCP_AO_INFO, &info, EINVAL, "set rnext key on a listen socket"); + + sk = prepare_defs(TCP_AO_INFO, &info); + make_listen(sk); + info.set_current = 1; + info.set_rnext = 1; + setsockopt_checked(sk, TCP_AO_INFO, &info, EINVAL, "set current+rnext key on a listen socket"); + + sk = prepare_defs(TCP_AO_INFO, &info); + info.reserved = 1; + setsockopt_checked(sk, TCP_AO_INFO, &info, EINVAL, "using reserved padding"); + + sk = prepare_defs(TCP_AO_INFO, &info); + info.reserved2 = 1; + setsockopt_checked(sk, TCP_AO_INFO, &info, EINVAL, "using reserved2 padding"); + + sk = prepare_defs(TCP_AO_INFO, &info); + info.accept_icmps = 1; + setsockopt_checked(sk, TCP_AO_INFO, &info, 0, "accept_icmps"); + + sk = prepare_defs(TCP_AO_INFO, &info); + info.ao_required = 1; + setsockopt_checked(sk, TCP_AO_INFO, &info, 0, "ao required"); + + if (!should_skip_test("ao required with MD5 key", KCONFIG_TCP_MD5)) { + sk = prepare_defs(TCP_AO_INFO, &info); + info.ao_required = 1; + if (test_set_md5(sk, tcp_md5_client, TEST_PREFIX, -1, + "long long secret")) { + test_error("setsockopt(TCP_MD5SIG_EXT)"); + close(sk); + } else { + setsockopt_checked(sk, TCP_AO_INFO, &info, EKEYREJECTED, + "ao required with MD5 key"); + } + } + + sk = prepare_defs(TCP_AO_INFO, &info); + info.set_current = 1; + setsockopt_checked(sk, TCP_AO_INFO, &info, ENOENT, "set non-existing current key"); + + sk = prepare_defs(TCP_AO_INFO, &info); + info.set_rnext = 1; + setsockopt_checked(sk, TCP_AO_INFO, &info, ENOENT, "set non-existing rnext key"); + + sk = prepare_defs(TCP_AO_INFO, &info); + info.set_current = 1; + info.set_rnext = 1; + setsockopt_checked(sk, TCP_AO_INFO, &info, ENOENT, "set non-existing current+rnext key"); + + sk = prepare_defs(TCP_AO_INFO, &info); + info.set_current = 1; + info.current_key = 100; + setsockopt_checked(sk, TCP_AO_INFO, &info, 0, "set current key"); + + sk = prepare_defs(TCP_AO_INFO, &info); + info.set_rnext = 1; + info.rnext = 100; + setsockopt_checked(sk, TCP_AO_INFO, &info, 0, "set rnext key"); + + sk = prepare_defs(TCP_AO_INFO, &info); + info.set_current = 1; + info.set_rnext = 1; + info.current_key = 100; + info.rnext = 100; + setsockopt_checked(sk, TCP_AO_INFO, &info, 0, "set current+rnext key"); + + sk = prepare_defs(TCP_AO_INFO, &info); + info.set_counters = 1; + info.pkt_good = 321; + info.pkt_bad = 888; + info.pkt_key_not_found = 654; + info.pkt_ao_required = 987654; + info.pkt_dropped_icmp = 10000; + setsockopt_checked(sk, TCP_AO_INFO, &info, 0, "set counters"); + + sk = prepare_defs(TCP_AO_INFO, &info); + setsockopt_checked(sk, TCP_AO_INFO, &info, 0, "no-op"); +} + +static void getsockopt_checked(int sk, struct tcp_ao_getsockopt *optval, + int err, const char *tst) +{ + socklen_t len = sizeof(struct tcp_ao_getsockopt); + + __setsockopt_checked(sk, TCP_AO_GET_KEYS, true, optval, &len, err, + "get keys: ", tst); +} + +static void test_einval_get_keys(void) +{ + struct tcp_ao_getsockopt out; + int sk; + + sk = socket(test_family, SOCK_STREAM, IPPROTO_TCP); + if (sk < 0) + test_error("socket()"); + getsockopt_checked(sk, &out, ENOENT, "no ao_info"); + + sk = prepare_defs(TCP_AO_GET_KEYS, &out); + getsockopt_checked(sk, &out, 0, "proper tcp_ao_get_mkts()"); + + sk = prepare_defs(TCP_AO_GET_KEYS, &out); + out.pkt_good = 643; + getsockopt_checked(sk, &out, EINVAL, "set out-only pkt_good counter"); + + sk = prepare_defs(TCP_AO_GET_KEYS, &out); + out.pkt_bad = 94; + getsockopt_checked(sk, &out, EINVAL, "set out-only pkt_bad counter"); + + sk = prepare_defs(TCP_AO_GET_KEYS, &out); + out.keyflags = (uint8_t)(-1); + getsockopt_checked(sk, &out, EINVAL, "bad keyflags"); + + sk = prepare_defs(TCP_AO_GET_KEYS, &out); + out.ifindex = 42; + getsockopt_checked(sk, &out, EINVAL, + "ifindex without TCP_AO_KEYF_IFNINDEX"); + + sk = prepare_defs(TCP_AO_GET_KEYS, &out); + out.reserved = 1; + getsockopt_checked(sk, &out, EINVAL, "using reserved field"); + + sk = prepare_defs(TCP_AO_GET_KEYS, &out); + out.get_all = 0; + out.prefix = 0; + tcp_addr_to_sockaddr_in(&out.addr, &this_ip_dest, 0); + getsockopt_checked(sk, &out, EINVAL, "no prefix, addr"); + + sk = prepare_defs(TCP_AO_GET_KEYS, &out); + out.get_all = 0; + out.prefix = 0; + memcpy(&out.addr, &SOCKADDR_ANY, sizeof(SOCKADDR_ANY)); + getsockopt_checked(sk, &out, 0, "no prefix, any addr"); + + sk = prepare_defs(TCP_AO_GET_KEYS, &out); + out.get_all = 0; + out.prefix = 32; + memcpy(&out.addr, &SOCKADDR_ANY, sizeof(SOCKADDR_ANY)); + getsockopt_checked(sk, &out, EINVAL, "prefix, any addr"); + + sk = prepare_defs(TCP_AO_GET_KEYS, &out); + out.get_all = 0; + out.prefix = 129; + tcp_addr_to_sockaddr_in(&out.addr, &this_ip_dest, 0); + getsockopt_checked(sk, &out, EINVAL, "too big prefix"); + + sk = prepare_defs(TCP_AO_GET_KEYS, &out); + out.get_all = 0; + out.prefix = 2; + tcp_addr_to_sockaddr_in(&out.addr, &this_ip_dest, 0); + getsockopt_checked(sk, &out, EINVAL, "too short prefix"); + + sk = prepare_defs(TCP_AO_GET_KEYS, &out); + out.get_all = 0; + out.prefix = DEFAULT_TEST_PREFIX; + tcp_addr_to_sockaddr_in(&out.addr, &this_ip_dest, 0); + getsockopt_checked(sk, &out, 0, "prefix + addr"); + + sk = prepare_defs(TCP_AO_GET_KEYS, &out); + out.get_all = 1; + out.prefix = DEFAULT_TEST_PREFIX; + getsockopt_checked(sk, &out, EINVAL, "get_all + prefix"); + + sk = prepare_defs(TCP_AO_GET_KEYS, &out); + out.get_all = 1; + tcp_addr_to_sockaddr_in(&out.addr, &this_ip_dest, 0); + getsockopt_checked(sk, &out, EINVAL, "get_all + addr"); + + sk = prepare_defs(TCP_AO_GET_KEYS, &out); + out.get_all = 1; + out.sndid = 1; + getsockopt_checked(sk, &out, EINVAL, "get_all + sndid"); + + sk = prepare_defs(TCP_AO_GET_KEYS, &out); + out.get_all = 1; + out.rcvid = 1; + getsockopt_checked(sk, &out, EINVAL, "get_all + rcvid"); + + sk = prepare_defs(TCP_AO_GET_KEYS, &out); + out.get_all = 0; + out.is_current = 1; + out.prefix = DEFAULT_TEST_PREFIX; + getsockopt_checked(sk, &out, EINVAL, "current + prefix"); + + sk = prepare_defs(TCP_AO_GET_KEYS, &out); + out.get_all = 0; + out.is_current = 1; + tcp_addr_to_sockaddr_in(&out.addr, &this_ip_dest, 0); + getsockopt_checked(sk, &out, EINVAL, "current + addr"); + + sk = prepare_defs(TCP_AO_GET_KEYS, &out); + out.get_all = 0; + out.is_current = 1; + out.sndid = 1; + getsockopt_checked(sk, &out, EINVAL, "current + sndid"); + + sk = prepare_defs(TCP_AO_GET_KEYS, &out); + out.get_all = 0; + out.is_current = 1; + out.rcvid = 1; + getsockopt_checked(sk, &out, EINVAL, "current + rcvid"); + + sk = prepare_defs(TCP_AO_GET_KEYS, &out); + out.get_all = 0; + out.is_rnext = 1; + out.prefix = DEFAULT_TEST_PREFIX; + getsockopt_checked(sk, &out, EINVAL, "rnext + prefix"); + + sk = prepare_defs(TCP_AO_GET_KEYS, &out); + out.get_all = 0; + out.is_rnext = 1; + tcp_addr_to_sockaddr_in(&out.addr, &this_ip_dest, 0); + getsockopt_checked(sk, &out, EINVAL, "rnext + addr"); + + sk = prepare_defs(TCP_AO_GET_KEYS, &out); + out.get_all = 0; + out.is_rnext = 1; + out.sndid = 1; + getsockopt_checked(sk, &out, EINVAL, "rnext + sndid"); + + sk = prepare_defs(TCP_AO_GET_KEYS, &out); + out.get_all = 0; + out.is_rnext = 1; + out.rcvid = 1; + getsockopt_checked(sk, &out, EINVAL, "rnext + rcvid"); + + sk = prepare_defs(TCP_AO_GET_KEYS, &out); + out.get_all = 1; + out.is_current = 1; + getsockopt_checked(sk, &out, EINVAL, "get_all + current"); + + sk = prepare_defs(TCP_AO_GET_KEYS, &out); + out.get_all = 1; + out.is_rnext = 1; + getsockopt_checked(sk, &out, EINVAL, "get_all + rnext"); + + sk = prepare_defs(TCP_AO_GET_KEYS, &out); + out.get_all = 0; + out.is_current = 1; + out.is_rnext = 1; + getsockopt_checked(sk, &out, 0, "current + rnext"); +} + +static void einval_tests(void) +{ + test_einval_add_key(); + test_einval_del_key(); + test_einval_ao_info(); + test_einval_get_keys(); +} + +static void duplicate_tests(void) +{ + union tcp_addr network_dup; + struct tcp_ao_add ao, ao2; + int sk; + + sk = prepare_defs(TCP_AO_ADD_KEY, &ao); + if (setsockopt(sk, IPPROTO_TCP, TCP_AO_ADD_KEY, &ao, sizeof(ao))) + test_error("setsockopt()"); + setsockopt_checked(sk, TCP_AO_ADD_KEY, &ao, EEXIST, "duplicate: full copy"); + + sk = prepare_defs(TCP_AO_ADD_KEY, &ao); + ao2 = ao; + memcpy(&ao2.addr, &SOCKADDR_ANY, sizeof(SOCKADDR_ANY)); + ao2.prefix = 0; + if (setsockopt(sk, IPPROTO_TCP, TCP_AO_ADD_KEY, &ao2, sizeof(ao))) + test_error("setsockopt()"); + setsockopt_checked(sk, TCP_AO_ADD_KEY, &ao, EEXIST, "duplicate: any addr key on the socket"); + + sk = prepare_defs(TCP_AO_ADD_KEY, &ao); + if (setsockopt(sk, IPPROTO_TCP, TCP_AO_ADD_KEY, &ao, sizeof(ao))) + test_error("setsockopt()"); + memcpy(&ao.addr, &SOCKADDR_ANY, sizeof(SOCKADDR_ANY)); + ao.prefix = 0; + setsockopt_checked(sk, TCP_AO_ADD_KEY, &ao, EEXIST, "duplicate: add any addr key"); + + if (inet_pton(TEST_FAMILY, TEST_NETWORK, &network_dup) != 1) + test_error("Can't convert ip address %s", TEST_NETWORK); + sk = prepare_defs(TCP_AO_ADD_KEY, &ao); + if (setsockopt(sk, IPPROTO_TCP, TCP_AO_ADD_KEY, &ao, sizeof(ao))) + test_error("setsockopt()"); + if (test_prepare_def_key(&ao, "password", 0, network_dup, + 16, 0, 100, 100)) + test_error("prepare default tcp_ao_add"); + setsockopt_checked(sk, TCP_AO_ADD_KEY, &ao, EEXIST, "duplicate: add any addr for the same subnet"); + + sk = prepare_defs(TCP_AO_ADD_KEY, &ao); + if (setsockopt(sk, IPPROTO_TCP, TCP_AO_ADD_KEY, &ao, sizeof(ao))) + test_error("setsockopt()"); + setsockopt_checked(sk, TCP_AO_ADD_KEY, &ao, EEXIST, "duplicate: full copy of a key"); + + sk = prepare_defs(TCP_AO_ADD_KEY, &ao); + if (setsockopt(sk, IPPROTO_TCP, TCP_AO_ADD_KEY, &ao, sizeof(ao))) + test_error("setsockopt()"); + ao.rcvid = 101; + setsockopt_checked(sk, TCP_AO_ADD_KEY, &ao, EEXIST, "duplicate: RecvID differs"); + + sk = prepare_defs(TCP_AO_ADD_KEY, &ao); + if (setsockopt(sk, IPPROTO_TCP, TCP_AO_ADD_KEY, &ao, sizeof(ao))) + test_error("setsockopt()"); + ao.sndid = 101; + setsockopt_checked(sk, TCP_AO_ADD_KEY, &ao, EEXIST, "duplicate: SendID differs"); +} + +static void *client_fn(void *arg) +{ + if (inet_pton(TEST_FAMILY, __TEST_CLIENT_IP(2), &tcp_md5_client) != 1) + test_error("Can't convert ip address"); + extend_tests(); + einval_tests(); + duplicate_tests(); + /* + * TODO: check getsockopt(TCP_AO_GET_KEYS) with different filters + * returning proper nr & keys; + */ + + return NULL; +} + +int main(int argc, char *argv[]) +{ + test_init(120, client_fn, NULL); + return 0; +} diff --git a/tools/testing/selftests/net/tcp_ao/unsigned-md5.c b/tools/testing/selftests/net/tcp_ao/unsigned-md5.c new file mode 100644 index 00000000000000..7cffde02d2be2e --- /dev/null +++ b/tools/testing/selftests/net/tcp_ao/unsigned-md5.c @@ -0,0 +1,742 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Author: Dmitry Safonov */ +#include +#include "aolib.h" + +#define fault(type) (inj == FAULT_ ## type) +static const char *md5_password = "Some evil genius, enemy to mankind, must have been the first contriver."; +static const char *ao_password = DEFAULT_TEST_PASSWORD; + +static union tcp_addr client2; +static union tcp_addr client3; + +static const int test_vrf_ifindex = 200; +static const uint8_t test_vrf_tabid = 42; +static void setup_vrfs(void) +{ + int err; + + if (!kernel_config_has(KCONFIG_NET_VRF)) + return; + + err = add_vrf("ksft-vrf", test_vrf_tabid, test_vrf_ifindex, -1); + if (err) + test_error("Failed to add a VRF: %d", err); + + err = link_set_up("ksft-vrf"); + if (err) + test_error("Failed to bring up a VRF"); + + err = ip_route_add_vrf(veth_name, TEST_FAMILY, + this_ip_addr, this_ip_dest, test_vrf_tabid); + if (err) + test_error("Failed to add a route to VRF"); +} + +static void try_accept(const char *tst_name, unsigned int port, + union tcp_addr *md5_addr, uint8_t md5_prefix, + union tcp_addr *ao_addr, uint8_t ao_prefix, + bool set_ao_required, + uint8_t sndid, uint8_t rcvid, uint8_t vrf, + const char *cnt_name, test_cnt cnt_expected, + int needs_tcp_md5, fault_t inj) +{ + struct tcp_ao_counters ao_cnt1, ao_cnt2; + uint64_t before_cnt = 0, after_cnt = 0; /* silence GCC */ + int lsk, err, sk = 0; + time_t timeout; + + if (needs_tcp_md5 && should_skip_test(tst_name, KCONFIG_TCP_MD5)) + return; + + lsk = test_listen_socket(this_ip_addr, port, 1); + + if (md5_addr && test_set_md5(lsk, *md5_addr, md5_prefix, -1, md5_password)) + test_error("setsockopt(TCP_MD5SIG_EXT)"); + + if (ao_addr && test_add_key(lsk, ao_password, + *ao_addr, ao_prefix, sndid, rcvid)) + test_error("setsockopt(TCP_AO_ADD_KEY)"); + + if (set_ao_required && test_set_ao_flags(lsk, true, false)) + test_error("setsockopt(TCP_AO_INFO)"); + + if (cnt_name) + before_cnt = netstat_get_one(cnt_name, NULL); + if (ao_addr && test_get_tcp_ao_counters(lsk, &ao_cnt1)) + test_error("test_get_tcp_ao_counters()"); + + synchronize_threads(); /* preparations done */ + + timeout = fault(TIMEOUT) ? TEST_RETRANSMIT_SEC : TEST_TIMEOUT_SEC; + err = test_wait_fd(lsk, timeout, 0); + if (err == -ETIMEDOUT) { + if (!fault(TIMEOUT)) + test_fail("timeouted for accept()"); + } else if (err < 0) { + test_error("test_wait_fd()"); + } else { + if (fault(TIMEOUT)) + test_fail("ready to accept"); + + sk = accept(lsk, NULL, NULL); + if (sk < 0) { + test_error("accept()"); + } else { + if (fault(TIMEOUT)) + test_fail("%s: accepted", tst_name); + } + } + + if (ao_addr && test_get_tcp_ao_counters(lsk, &ao_cnt2)) + test_error("test_get_tcp_ao_counters()"); + close(lsk); + + if (!cnt_name) { + test_ok("%s: no counter checks", tst_name); + goto out; + } + + after_cnt = netstat_get_one(cnt_name, NULL); + + if (after_cnt <= before_cnt) { + test_fail("%s: %s counter did not increase: %zu <= %zu", + tst_name, cnt_name, after_cnt, before_cnt); + } else { + test_ok("%s: counter %s increased %zu => %zu", + tst_name, cnt_name, before_cnt, after_cnt); + } + if (ao_addr) + test_tcp_ao_counters_cmp(tst_name, &ao_cnt1, &ao_cnt2, cnt_expected); + +out: + synchronize_threads(); /* close() */ + if (sk > 0) + close(sk); +} + +static void server_add_routes(void) +{ + int family = TEST_FAMILY; + + synchronize_threads(); /* client_add_ips() */ + + if (ip_route_add(veth_name, family, this_ip_addr, client2)) + test_error("Failed to add route"); + if (ip_route_add(veth_name, family, this_ip_addr, client3)) + test_error("Failed to add route"); +} + +static void server_add_fail_tests(unsigned int *port) +{ + union tcp_addr addr_any = {}; + + try_accept("TCP-AO established: add TCP-MD5 key", (*port)++, NULL, 0, + &addr_any, 0, 0, 100, 100, 0, "TCPAOGood", TEST_CNT_GOOD, + 1, 0); + try_accept("TCP-MD5 established: add TCP-AO key", (*port)++, &addr_any, + 0, NULL, 0, 0, 0, 0, 0, NULL, 0, 1, 0); + try_accept("non-signed established: add TCP-AO key", (*port)++, NULL, 0, + NULL, 0, 0, 0, 0, 0, "CurrEstab", 0, 0, 0); +} + +static void server_vrf_tests(unsigned int *port) +{ + setup_vrfs(); +} + +static void *server_fn(void *arg) +{ + unsigned int port = test_server_port; + union tcp_addr addr_any = {}; + + server_add_routes(); + + try_accept("AO server (INADDR_ANY): AO client", port++, NULL, 0, + &addr_any, 0, 0, 100, 100, 0, "TCPAOGood", + TEST_CNT_GOOD, 0, 0); + try_accept("AO server (INADDR_ANY): MD5 client", port++, NULL, 0, + &addr_any, 0, 0, 100, 100, 0, "TCPMD5Unexpected", + 0, 1, FAULT_TIMEOUT); + try_accept("AO server (INADDR_ANY): no sign client", port++, NULL, 0, + &addr_any, 0, 0, 100, 100, 0, "TCPAORequired", + TEST_CNT_AO_REQUIRED, 0, FAULT_TIMEOUT); + try_accept("AO server (AO_REQUIRED): AO client", port++, NULL, 0, + &this_ip_dest, TEST_PREFIX, true, + 100, 100, 0, "TCPAOGood", TEST_CNT_GOOD, 0, 0); + try_accept("AO server (AO_REQUIRED): unsigned client", port++, NULL, 0, + &this_ip_dest, TEST_PREFIX, true, + 100, 100, 0, "TCPAORequired", + TEST_CNT_AO_REQUIRED, 0, FAULT_TIMEOUT); + + try_accept("MD5 server (INADDR_ANY): AO client", port++, &addr_any, 0, + NULL, 0, 0, 0, 0, 0, "TCPAOKeyNotFound", + 0, 1, FAULT_TIMEOUT); + try_accept("MD5 server (INADDR_ANY): MD5 client", port++, &addr_any, 0, + NULL, 0, 0, 0, 0, 0, NULL, 0, 1, 0); + try_accept("MD5 server (INADDR_ANY): no sign client", port++, &addr_any, + 0, NULL, 0, 0, 0, 0, 0, "TCPMD5NotFound", + 0, 1, FAULT_TIMEOUT); + + try_accept("no sign server: AO client", port++, NULL, 0, + NULL, 0, 0, 0, 0, 0, "TCPAOKeyNotFound", + TEST_CNT_AO_KEY_NOT_FOUND, 0, FAULT_TIMEOUT); + try_accept("no sign server: MD5 client", port++, NULL, 0, + NULL, 0, 0, 0, 0, 0, "TCPMD5Unexpected", + 0, 1, FAULT_TIMEOUT); + try_accept("no sign server: no sign client", port++, NULL, 0, + NULL, 0, 0, 0, 0, 0, "CurrEstab", 0, 0, 0); + + try_accept("AO+MD5 server: AO client (matching)", port++, + &this_ip_dest, TEST_PREFIX, &client2, TEST_PREFIX, 0, + 100, 100, 0, "TCPAOGood", TEST_CNT_GOOD, 1, 0); + try_accept("AO+MD5 server: AO client (misconfig, matching MD5)", port++, + &this_ip_dest, TEST_PREFIX, &client2, TEST_PREFIX, 0, + 100, 100, 0, "TCPAOKeyNotFound", TEST_CNT_AO_KEY_NOT_FOUND, + 1, FAULT_TIMEOUT); + try_accept("AO+MD5 server: AO client (misconfig, non-matching)", port++, + &this_ip_dest, TEST_PREFIX, &client2, TEST_PREFIX, 0, + 100, 100, 0, "TCPAOKeyNotFound", TEST_CNT_AO_KEY_NOT_FOUND, + 1, FAULT_TIMEOUT); + try_accept("AO+MD5 server: MD5 client (matching)", port++, + &this_ip_dest, TEST_PREFIX, &client2, TEST_PREFIX, 0, + 100, 100, 0, NULL, 0, 1, 0); + try_accept("AO+MD5 server: MD5 client (misconfig, matching AO)", port++, + &this_ip_dest, TEST_PREFIX, &client2, TEST_PREFIX, 0, + 100, 100, 0, "TCPMD5Unexpected", 0, 1, FAULT_TIMEOUT); + try_accept("AO+MD5 server: MD5 client (misconfig, non-matching)", port++, + &this_ip_dest, TEST_PREFIX, &client2, TEST_PREFIX, 0, + 100, 100, 0, "TCPMD5Unexpected", 0, 1, FAULT_TIMEOUT); + try_accept("AO+MD5 server: no sign client (unmatched)", port++, + &this_ip_dest, TEST_PREFIX, &client2, TEST_PREFIX, 0, + 100, 100, 0, "CurrEstab", 0, 1, 0); + try_accept("AO+MD5 server: no sign client (misconfig, matching AO)", + port++, &this_ip_dest, TEST_PREFIX, &client2, TEST_PREFIX, 0, + 100, 100, 0, "TCPAORequired", + TEST_CNT_AO_REQUIRED, 1, FAULT_TIMEOUT); + try_accept("AO+MD5 server: no sign client (misconfig, matching MD5)", + port++, &this_ip_dest, TEST_PREFIX, &client2, TEST_PREFIX, 0, + 100, 100, 0, "TCPMD5NotFound", 0, 1, FAULT_TIMEOUT); + + try_accept("AO+MD5 server: client with both [TCP-MD5] and TCP-AO keys", + port++, &this_ip_dest, TEST_PREFIX, &client2, TEST_PREFIX, 0, + 100, 100, 0, NULL, 0, 1, FAULT_TIMEOUT); + try_accept("AO+MD5 server: client with both TCP-MD5 and [TCP-AO] keys", + port++, &this_ip_dest, TEST_PREFIX, &client2, TEST_PREFIX, 0, + 100, 100, 0, NULL, 0, 1, FAULT_TIMEOUT); + + server_add_fail_tests(&port); + + server_vrf_tests(&port); + + /* client exits */ + synchronize_threads(); + return NULL; +} + +static int client_bind(int sk, union tcp_addr bind_addr) +{ +#ifdef IPV6_TEST + struct sockaddr_in6 addr = { + .sin6_family = AF_INET6, + .sin6_port = 0, + .sin6_addr = bind_addr.a6, + }; +#else + struct sockaddr_in addr = { + .sin_family = AF_INET, + .sin_port = 0, + .sin_addr = bind_addr.a4, + }; +#endif + return bind(sk, &addr, sizeof(addr)); +} + +static void try_connect(const char *tst_name, unsigned int port, + union tcp_addr *md5_addr, uint8_t md5_prefix, + union tcp_addr *ao_addr, uint8_t ao_prefix, + uint8_t sndid, uint8_t rcvid, uint8_t vrf, + fault_t inj, int needs_tcp_md5, union tcp_addr *bind_addr) +{ + time_t timeout; + int sk, ret; + + if (needs_tcp_md5 && should_skip_test(tst_name, KCONFIG_TCP_MD5)) + return; + + sk = socket(test_family, SOCK_STREAM, IPPROTO_TCP); + if (sk < 0) + test_error("socket()"); + + if (bind_addr && client_bind(sk, *bind_addr)) + test_error("bind()"); + + if (md5_addr && test_set_md5(sk, *md5_addr, md5_prefix, -1, md5_password)) + test_error("setsockopt(TCP_MD5SIG_EXT)"); + + if (ao_addr && test_add_key(sk, ao_password, *ao_addr, + ao_prefix, sndid, rcvid)) + test_error("setsockopt(TCP_AO_ADD_KEY)"); + + synchronize_threads(); /* preparations done */ + + timeout = fault(TIMEOUT) ? TEST_RETRANSMIT_SEC : TEST_TIMEOUT_SEC; + ret = _test_connect_socket(sk, this_ip_dest, port, timeout); + + if (ret < 0) { + if (fault(KEYREJECT) && ret == -EKEYREJECTED) + test_ok("%s: connect() was prevented", tst_name); + else if (ret == -ETIMEDOUT && fault(TIMEOUT)) + test_ok("%s", tst_name); + else if (ret == -ECONNREFUSED && + (fault(TIMEOUT) || fault(KEYREJECT))) + test_ok("%s: refused to connect", tst_name); + else + test_error("%s: connect() returned %d", tst_name, ret); + goto out; + } + + if (fault(TIMEOUT) || fault(KEYREJECT)) + test_fail("%s: connected", tst_name); + else + test_ok("%s: connected", tst_name); + +out: + synchronize_threads(); /* close() */ + /* _test_connect_socket() cleans up on failure */ + if (ret > 0) + close(sk); +} + +#define PREINSTALL_MD5_FIRST BIT(0) +#define PREINSTALL_AO BIT(1) +#define POSTINSTALL_AO BIT(2) +#define PREINSTALL_MD5 BIT(3) +#define POSTINSTALL_MD5 BIT(4) + +static int try_add_key_vrf(int sk, union tcp_addr in_addr, uint8_t prefix, + int vrf, uint8_t sndid, uint8_t rcvid, + bool set_ao_required) +{ + uint8_t keyflags = 0; + + if (vrf >= 0) + keyflags |= TCP_AO_KEYF_IFINDEX; + else + vrf = 0; + if (set_ao_required) { + int err = test_set_ao_flags(sk, true, 0); + + if (err) + return err; + } + return test_add_key_vrf(sk, ao_password, keyflags, in_addr, prefix, + (uint8_t)vrf, sndid, rcvid); +} + +static bool test_continue(const char *tst_name, int err, + fault_t inj, bool added_ao) +{ + bool expected_to_fail; + + expected_to_fail = fault(PREINSTALL_AO) && added_ao; + expected_to_fail |= fault(PREINSTALL_MD5) && !added_ao; + + if (!err) { + if (!expected_to_fail) + return true; + test_fail("%s: setsockopt()s were expected to fail", tst_name); + return false; + } + if (err != -EKEYREJECTED || !expected_to_fail) { + test_error("%s: setsockopt(%s) = %d", tst_name, + added_ao ? "TCP_AO_ADD_KEY" : "TCP_MD5SIG_EXT", err); + return false; + } + test_ok("%s: prefailed as expected: %m", tst_name); + return false; +} + +static int open_add(const char *tst_name, unsigned int port, + unsigned int strategy, + union tcp_addr md5_addr, uint8_t md5_prefix, int md5_vrf, + union tcp_addr ao_addr, uint8_t ao_prefix, + int ao_vrf, bool set_ao_required, + uint8_t sndid, uint8_t rcvid, + fault_t inj) +{ + int sk; + + sk = socket(test_family, SOCK_STREAM, IPPROTO_TCP); + if (sk < 0) + test_error("socket()"); + + if (client_bind(sk, this_ip_addr)) + test_error("bind()"); + + if (strategy & PREINSTALL_MD5_FIRST) { + if (test_set_md5(sk, md5_addr, md5_prefix, md5_vrf, md5_password)) + test_error("setsockopt(TCP_MD5SIG_EXT)"); + } + + if (strategy & PREINSTALL_AO) { + int err = try_add_key_vrf(sk, ao_addr, ao_prefix, ao_vrf, + sndid, rcvid, set_ao_required); + + if (!test_continue(tst_name, err, inj, true)) { + close(sk); + return -1; + } + } + + if (strategy & PREINSTALL_MD5) { + errno = 0; + test_set_md5(sk, md5_addr, md5_prefix, md5_vrf, md5_password); + if (!test_continue(tst_name, -errno, inj, false)) { + close(sk); + return -1; + } + } + + return sk; +} + +static void try_to_preadd(const char *tst_name, unsigned int port, + unsigned int strategy, + union tcp_addr md5_addr, uint8_t md5_prefix, + int md5_vrf, + union tcp_addr ao_addr, uint8_t ao_prefix, + int ao_vrf, bool set_ao_required, + uint8_t sndid, uint8_t rcvid, + int needs_tcp_md5, int needs_vrf, fault_t inj) +{ + int sk; + + if (needs_tcp_md5 && should_skip_test(tst_name, KCONFIG_TCP_MD5)) + return; + if (needs_vrf && should_skip_test(tst_name, KCONFIG_NET_VRF)) + return; + + sk = open_add(tst_name, port, strategy, md5_addr, md5_prefix, md5_vrf, + ao_addr, ao_prefix, ao_vrf, set_ao_required, + sndid, rcvid, inj); + if (sk < 0) + return; + + test_ok("%s", tst_name); + close(sk); +} + +static void try_to_add(const char *tst_name, unsigned int port, + unsigned int strategy, + union tcp_addr md5_addr, uint8_t md5_prefix, + int md5_vrf, + union tcp_addr ao_addr, uint8_t ao_prefix, + int ao_vrf, uint8_t sndid, uint8_t rcvid, + int needs_tcp_md5, fault_t inj) +{ + time_t timeout; + int sk, ret; + + if (needs_tcp_md5 && should_skip_test(tst_name, KCONFIG_TCP_MD5)) + return; + + sk = open_add(tst_name, port, strategy, md5_addr, md5_prefix, md5_vrf, + ao_addr, ao_prefix, ao_vrf, 0, sndid, rcvid, inj); + if (sk < 0) + return; + + synchronize_threads(); /* preparations done */ + + timeout = fault(TIMEOUT) ? TEST_RETRANSMIT_SEC : TEST_TIMEOUT_SEC; + ret = _test_connect_socket(sk, this_ip_dest, port, timeout); + + if (ret <= 0) { + test_error("%s: connect() returned %d", tst_name, ret); + goto out; + } + + if (strategy & POSTINSTALL_MD5) { + if (test_set_md5(sk, md5_addr, md5_prefix, md5_vrf, md5_password)) { + if (fault(POSTINSTALL)) { + test_ok("%s: postfailed as expected", tst_name); + goto out; + } else { + test_error("setsockopt(TCP_MD5SIG_EXT)"); + } + } else if (fault(POSTINSTALL)) { + test_fail("%s: post setsockopt() was expected to fail", tst_name); + goto out; + } + } + + if (strategy & POSTINSTALL_AO) { + if (try_add_key_vrf(sk, ao_addr, ao_prefix, ao_vrf, + sndid, rcvid, 0)) { + if (fault(POSTINSTALL)) { + test_ok("%s: postfailed as expected", tst_name); + goto out; + } else { + test_error("setsockopt(TCP_AO_ADD_KEY)"); + } + } else if (fault(POSTINSTALL)) { + test_fail("%s: post setsockopt() was expected to fail", tst_name); + goto out; + } + } + +out: + synchronize_threads(); /* close() */ + /* _test_connect_socket() cleans up on failure */ + if (ret > 0) + close(sk); +} + +static void client_add_ip(union tcp_addr *client, const char *ip) +{ + int family = TEST_FAMILY; + + if (inet_pton(family, ip, client) != 1) + test_error("Can't convert ip address %s", ip); + + if (ip_addr_add(veth_name, family, *client, TEST_PREFIX)) + test_error("Failed to add ip address"); + if (ip_route_add(veth_name, family, *client, this_ip_dest)) + test_error("Failed to add route"); +} + +static void client_add_ips(void) +{ + client_add_ip(&client2, __TEST_CLIENT_IP(2)); + client_add_ip(&client3, __TEST_CLIENT_IP(3)); + synchronize_threads(); /* server_add_routes() */ +} + +static void client_add_fail_tests(unsigned int *port) +{ + try_to_add("TCP-AO established: add TCP-MD5 key", + (*port)++, POSTINSTALL_MD5 | PREINSTALL_AO, + this_ip_dest, TEST_PREFIX, -1, this_ip_dest, TEST_PREFIX, 0, + 100, 100, 1, FAULT_POSTINSTALL); + try_to_add("TCP-MD5 established: add TCP-AO key", + (*port)++, PREINSTALL_MD5 | POSTINSTALL_AO, + this_ip_dest, TEST_PREFIX, -1, this_ip_dest, TEST_PREFIX, 0, + 100, 100, 1, FAULT_POSTINSTALL); + try_to_add("non-signed established: add TCP-AO key", + (*port)++, POSTINSTALL_AO, + this_ip_dest, TEST_PREFIX, -1, this_ip_dest, TEST_PREFIX, 0, + 100, 100, 0, FAULT_POSTINSTALL); + + try_to_add("TCP-AO key intersects with existing TCP-MD5 key", + (*port)++, PREINSTALL_MD5_FIRST | PREINSTALL_AO, + this_ip_addr, TEST_PREFIX, -1, this_ip_addr, TEST_PREFIX, -1, + 100, 100, 1, FAULT_PREINSTALL_AO); + try_to_add("TCP-MD5 key intersects with existing TCP-AO key", + (*port)++, PREINSTALL_MD5 | PREINSTALL_AO, + this_ip_addr, TEST_PREFIX, -1, this_ip_addr, TEST_PREFIX, -1, + 100, 100, 1, FAULT_PREINSTALL_MD5); + + try_to_preadd("TCP-MD5 key + TCP-AO required", + (*port)++, PREINSTALL_MD5_FIRST | PREINSTALL_AO, + this_ip_addr, TEST_PREFIX, -1, + this_ip_addr, TEST_PREFIX, -1, true, + 100, 100, 1, 0, FAULT_PREINSTALL_AO); + try_to_preadd("TCP-AO required on socket + TCP-MD5 key", + (*port)++, PREINSTALL_MD5 | PREINSTALL_AO, + this_ip_addr, TEST_PREFIX, -1, + this_ip_addr, TEST_PREFIX, -1, true, + 100, 100, 1, 0, FAULT_PREINSTALL_MD5); +} + +static void client_vrf_tests(unsigned int *port) +{ + setup_vrfs(); + + /* The following restrictions for setsockopt()s are expected: + * + * |--------------|-----------------|-------------|-------------| + * | | MD5 key without | MD5 key | MD5 key | + * | | l3index | l3index=0 | l3index=N | + * |--------------|-----------------|-------------|-------------| + * | TCP-AO key | | | | + * | without | reject | reject | reject | + * | l3index | | | | + * |--------------|-----------------|-------------|-------------| + * | TCP-AO key | | | | + * | l3index=0 | reject | reject | allow | + * |--------------|-----------------|-------------|-------------| + * | TCP-AO key | | | | + * | l3index=N | reject | allow | reject | + * |--------------|-----------------|-------------|-------------| + */ + try_to_preadd("VRF: TCP-AO key (no l3index) + TCP-MD5 key (no l3index)", + (*port)++, PREINSTALL_MD5 | PREINSTALL_AO, + this_ip_addr, TEST_PREFIX, -1, + this_ip_addr, TEST_PREFIX, -1, 0, 100, 100, + 1, 1, FAULT_PREINSTALL_MD5); + try_to_preadd("VRF: TCP-MD5 key (no l3index) + TCP-AO key (no l3index)", + (*port)++, PREINSTALL_MD5_FIRST | PREINSTALL_AO, + this_ip_addr, TEST_PREFIX, -1, + this_ip_addr, TEST_PREFIX, -1, 0, 100, 100, + 1, 1, FAULT_PREINSTALL_AO); + try_to_preadd("VRF: TCP-AO key (no l3index) + TCP-MD5 key (l3index=0)", + (*port)++, PREINSTALL_MD5 | PREINSTALL_AO, + this_ip_addr, TEST_PREFIX, 0, + this_ip_addr, TEST_PREFIX, -1, 0, 100, 100, + 1, 1, FAULT_PREINSTALL_MD5); + try_to_preadd("VRF: TCP-MD5 key (l3index=0) + TCP-AO key (no l3index)", + (*port)++, PREINSTALL_MD5_FIRST | PREINSTALL_AO, + this_ip_addr, TEST_PREFIX, 0, + this_ip_addr, TEST_PREFIX, -1, 0, 100, 100, + 1, 1, FAULT_PREINSTALL_AO); + try_to_preadd("VRF: TCP-AO key (no l3index) + TCP-MD5 key (l3index=N)", + (*port)++, PREINSTALL_MD5 | PREINSTALL_AO, + this_ip_addr, TEST_PREFIX, test_vrf_ifindex, + this_ip_addr, TEST_PREFIX, -1, 0, 100, 100, + 1, 1, FAULT_PREINSTALL_MD5); + try_to_preadd("VRF: TCP-MD5 key (l3index=N) + TCP-AO key (no l3index)", + (*port)++, PREINSTALL_MD5_FIRST | PREINSTALL_AO, + this_ip_addr, TEST_PREFIX, test_vrf_ifindex, + this_ip_addr, TEST_PREFIX, -1, 0, 100, 100, + 1, 1, FAULT_PREINSTALL_AO); + + try_to_preadd("VRF: TCP-AO key (l3index=0) + TCP-MD5 key (no l3index)", + (*port)++, PREINSTALL_MD5 | PREINSTALL_AO, + this_ip_addr, TEST_PREFIX, -1, + this_ip_addr, TEST_PREFIX, 0, 0, 100, 100, + 1, 1, FAULT_PREINSTALL_MD5); + try_to_preadd("VRF: TCP-MD5 key (no l3index) + TCP-AO key (l3index=0)", + (*port)++, PREINSTALL_MD5_FIRST | PREINSTALL_AO, + this_ip_addr, TEST_PREFIX, -1, + this_ip_addr, TEST_PREFIX, 0, 0, 100, 100, + 1, 1, FAULT_PREINSTALL_AO); + try_to_preadd("VRF: TCP-AO key (l3index=0) + TCP-MD5 key (l3index=0)", + (*port)++, PREINSTALL_MD5 | PREINSTALL_AO, + this_ip_addr, TEST_PREFIX, 0, + this_ip_addr, TEST_PREFIX, 0, 0, 100, 100, + 1, 1, FAULT_PREINSTALL_MD5); + try_to_preadd("VRF: TCP-MD5 key (l3index=0) + TCP-AO key (l3index=0)", + (*port)++, PREINSTALL_MD5_FIRST | PREINSTALL_AO, + this_ip_addr, TEST_PREFIX, 0, + this_ip_addr, TEST_PREFIX, 0, 0, 100, 100, + 1, 1, FAULT_PREINSTALL_AO); + try_to_preadd("VRF: TCP-AO key (l3index=0) + TCP-MD5 key (l3index=N)", + (*port)++, PREINSTALL_MD5 | PREINSTALL_AO, + this_ip_addr, TEST_PREFIX, test_vrf_ifindex, + this_ip_addr, TEST_PREFIX, 0, 0, 100, 100, + 1, 1, 0); + try_to_preadd("VRF: TCP-MD5 key (l3index=N) + TCP-AO key (l3index=0)", + (*port)++, PREINSTALL_MD5_FIRST | PREINSTALL_AO, + this_ip_addr, TEST_PREFIX, test_vrf_ifindex, + this_ip_addr, TEST_PREFIX, 0, 0, 100, 100, + 1, 1, 0); + + try_to_preadd("VRF: TCP-AO key (l3index=N) + TCP-MD5 key (no l3index)", + (*port)++, PREINSTALL_MD5 | PREINSTALL_AO, + this_ip_addr, TEST_PREFIX, test_vrf_ifindex, + this_ip_addr, TEST_PREFIX, -1, 0, 100, 100, + 1, 1, FAULT_PREINSTALL_MD5); + try_to_preadd("VRF: TCP-MD5 key (no l3index) + TCP-AO key (l3index=N)", + (*port)++, PREINSTALL_MD5_FIRST | PREINSTALL_AO, + this_ip_addr, TEST_PREFIX, -1, + this_ip_addr, TEST_PREFIX, test_vrf_ifindex, 0, 100, 100, + 1, 1, FAULT_PREINSTALL_AO); + try_to_preadd("VRF: TCP-AO key (l3index=N) + TCP-MD5 key (l3index=0)", + (*port)++, PREINSTALL_MD5 | PREINSTALL_AO, + this_ip_addr, TEST_PREFIX, 0, + this_ip_addr, TEST_PREFIX, test_vrf_ifindex, 0, 100, 100, + 1, 1, 0); + try_to_preadd("VRF: TCP-MD5 key (l3index=0) + TCP-AO key (l3index=N)", + (*port)++, PREINSTALL_MD5_FIRST | PREINSTALL_AO, + this_ip_addr, TEST_PREFIX, 0, + this_ip_addr, TEST_PREFIX, test_vrf_ifindex, 0, 100, 100, + 1, 1, 0); + try_to_preadd("VRF: TCP-AO key (l3index=N) + TCP-MD5 key (l3index=N)", + (*port)++, PREINSTALL_MD5 | PREINSTALL_AO, + this_ip_addr, TEST_PREFIX, test_vrf_ifindex, + this_ip_addr, TEST_PREFIX, test_vrf_ifindex, 0, 100, 100, + 1, 1, FAULT_PREINSTALL_MD5); + try_to_preadd("VRF: TCP-MD5 key (l3index=N) + TCP-AO key (l3index=N)", + (*port)++, PREINSTALL_MD5_FIRST | PREINSTALL_AO, + this_ip_addr, TEST_PREFIX, test_vrf_ifindex, + this_ip_addr, TEST_PREFIX, test_vrf_ifindex, 0, 100, 100, + 1, 1, FAULT_PREINSTALL_AO); +} + +static void *client_fn(void *arg) +{ + unsigned int port = test_server_port; + union tcp_addr addr_any = {}; + + client_add_ips(); + + try_connect("AO server (INADDR_ANY): AO client", port++, NULL, 0, + &addr_any, 0, 100, 100, 0, 0, 0, &this_ip_addr); + try_connect("AO server (INADDR_ANY): MD5 client", port++, &addr_any, 0, + NULL, 0, 100, 100, 0, FAULT_TIMEOUT, 1, &this_ip_addr); + try_connect("AO server (INADDR_ANY): unsigned client", port++, NULL, 0, + NULL, 0, 100, 100, 0, FAULT_TIMEOUT, 0, &this_ip_addr); + try_connect("AO server (AO_REQUIRED): AO client", port++, NULL, 0, + &addr_any, 0, 100, 100, 0, 0, 0, &this_ip_addr); + try_connect("AO server (AO_REQUIRED): unsigned client", port++, NULL, 0, + NULL, 0, 100, 100, 0, FAULT_TIMEOUT, 0, &client2); + + try_connect("MD5 server (INADDR_ANY): AO client", port++, NULL, 0, + &addr_any, 0, 100, 100, 0, FAULT_TIMEOUT, 1, &this_ip_addr); + try_connect("MD5 server (INADDR_ANY): MD5 client", port++, &addr_any, 0, + NULL, 0, 100, 100, 0, 0, 1, &this_ip_addr); + try_connect("MD5 server (INADDR_ANY): no sign client", port++, NULL, 0, + NULL, 0, 100, 100, 0, FAULT_TIMEOUT, 1, &this_ip_addr); + + try_connect("no sign server: AO client", port++, NULL, 0, + &addr_any, 0, 100, 100, 0, FAULT_TIMEOUT, 0, &this_ip_addr); + try_connect("no sign server: MD5 client", port++, &addr_any, 0, + NULL, 0, 100, 100, 0, FAULT_TIMEOUT, 1, &this_ip_addr); + try_connect("no sign server: no sign client", port++, NULL, 0, + NULL, 0, 100, 100, 0, 0, 0, &this_ip_addr); + + try_connect("AO+MD5 server: AO client (matching)", port++, NULL, 0, + &addr_any, 0, 100, 100, 0, 0, 1, &client2); + try_connect("AO+MD5 server: AO client (misconfig, matching MD5)", + port++, NULL, 0, &addr_any, 0, 100, 100, 0, + FAULT_TIMEOUT, 1, &this_ip_addr); + try_connect("AO+MD5 server: AO client (misconfig, non-matching)", + port++, NULL, 0, &addr_any, 0, 100, 100, 0, + FAULT_TIMEOUT, 1, &client3); + try_connect("AO+MD5 server: MD5 client (matching)", port++, &addr_any, 0, + NULL, 0, 100, 100, 0, 0, 1, &this_ip_addr); + try_connect("AO+MD5 server: MD5 client (misconfig, matching AO)", + port++, &addr_any, 0, NULL, 0, 100, 100, 0, FAULT_TIMEOUT, + 1, &client2); + try_connect("AO+MD5 server: MD5 client (misconfig, non-matching)", + port++, &addr_any, 0, NULL, 0, 100, 100, 0, FAULT_TIMEOUT, + 1, &client3); + try_connect("AO+MD5 server: no sign client (unmatched)", + port++, NULL, 0, NULL, 0, 100, 100, 0, 0, 1, &client3); + try_connect("AO+MD5 server: no sign client (misconfig, matching AO)", + port++, NULL, 0, NULL, 0, 100, 100, 0, FAULT_TIMEOUT, + 1, &client2); + try_connect("AO+MD5 server: no sign client (misconfig, matching MD5)", + port++, NULL, 0, NULL, 0, 100, 100, 0, FAULT_TIMEOUT, + 1, &this_ip_addr); + + try_connect("AO+MD5 server: client with both [TCP-MD5] and TCP-AO keys", + port++, &this_ip_addr, TEST_PREFIX, + &client2, TEST_PREFIX, 100, 100, 0, FAULT_KEYREJECT, + 1, &this_ip_addr); + try_connect("AO+MD5 server: client with both TCP-MD5 and [TCP-AO] keys", + port++, &this_ip_addr, TEST_PREFIX, + &client2, TEST_PREFIX, 100, 100, 0, FAULT_KEYREJECT, + 1, &client2); + + client_add_fail_tests(&port); + client_vrf_tests(&port); + + return NULL; +} + +int main(int argc, char *argv[]) +{ + test_init(72, server_fn, client_fn); + return 0; +} diff --git a/tools/testing/vsock/vsock_test.c b/tools/testing/vsock/vsock_test.c index 01fa816868bc4c..66246d81d65499 100644 --- a/tools/testing/vsock/vsock_test.c +++ b/tools/testing/vsock/vsock_test.c @@ -1232,6 +1232,171 @@ static void test_double_bind_connect_client(const struct test_opts *opts) } } +#define RCVLOWAT_CREDIT_UPD_BUF_SIZE (1024 * 128) +/* This define is the same as in 'include/linux/virtio_vsock.h': + * it is used to decide when to send credit update message during + * reading from rx queue of a socket. Value and its usage in + * kernel is important for this test. + */ +#define VIRTIO_VSOCK_MAX_PKT_BUF_SIZE (1024 * 64) + +static void test_stream_rcvlowat_def_cred_upd_client(const struct test_opts *opts) +{ + size_t buf_size; + void *buf; + int fd; + + fd = vsock_stream_connect(opts->peer_cid, 1234); + if (fd < 0) { + perror("connect"); + exit(EXIT_FAILURE); + } + + /* Send 1 byte more than peer's buffer size. */ + buf_size = RCVLOWAT_CREDIT_UPD_BUF_SIZE + 1; + + buf = malloc(buf_size); + if (!buf) { + perror("malloc"); + exit(EXIT_FAILURE); + } + + /* Wait until peer sets needed buffer size. */ + recv_byte(fd, 1, 0); + + if (send(fd, buf, buf_size, 0) != buf_size) { + perror("send failed"); + exit(EXIT_FAILURE); + } + + free(buf); + close(fd); +} + +static void test_stream_credit_update_test(const struct test_opts *opts, + bool low_rx_bytes_test) +{ + size_t recv_buf_size; + struct pollfd fds; + size_t buf_size; + void *buf; + int fd; + + fd = vsock_stream_accept(VMADDR_CID_ANY, 1234, NULL); + if (fd < 0) { + perror("accept"); + exit(EXIT_FAILURE); + } + + buf_size = RCVLOWAT_CREDIT_UPD_BUF_SIZE; + + if (setsockopt(fd, AF_VSOCK, SO_VM_SOCKETS_BUFFER_SIZE, + &buf_size, sizeof(buf_size))) { + perror("setsockopt(SO_VM_SOCKETS_BUFFER_SIZE)"); + exit(EXIT_FAILURE); + } + + if (low_rx_bytes_test) { + /* Set new SO_RCVLOWAT here. This enables sending credit + * update when number of bytes if our rx queue become < + * SO_RCVLOWAT value. + */ + recv_buf_size = 1 + VIRTIO_VSOCK_MAX_PKT_BUF_SIZE; + + if (setsockopt(fd, SOL_SOCKET, SO_RCVLOWAT, + &recv_buf_size, sizeof(recv_buf_size))) { + perror("setsockopt(SO_RCVLOWAT)"); + exit(EXIT_FAILURE); + } + } + + /* Send one dummy byte here, because 'setsockopt()' above also + * sends special packet which tells sender to update our buffer + * size. This 'send_byte()' will serialize such packet with data + * reads in a loop below. Sender starts transmission only when + * it receives this single byte. + */ + send_byte(fd, 1, 0); + + buf = malloc(buf_size); + if (!buf) { + perror("malloc"); + exit(EXIT_FAILURE); + } + + /* Wait until there will be 128KB of data in rx queue. */ + while (1) { + ssize_t res; + + res = recv(fd, buf, buf_size, MSG_PEEK); + if (res == buf_size) + break; + + if (res <= 0) { + fprintf(stderr, "unexpected 'recv()' return: %zi\n", res); + exit(EXIT_FAILURE); + } + } + + /* There is 128KB of data in the socket's rx queue, dequeue first + * 64KB, credit update is sent if 'low_rx_bytes_test' == true. + * Otherwise, credit update is sent in 'if (!low_rx_bytes_test)'. + */ + recv_buf_size = VIRTIO_VSOCK_MAX_PKT_BUF_SIZE; + recv_buf(fd, buf, recv_buf_size, 0, recv_buf_size); + + if (!low_rx_bytes_test) { + recv_buf_size++; + + /* Updating SO_RCVLOWAT will send credit update. */ + if (setsockopt(fd, SOL_SOCKET, SO_RCVLOWAT, + &recv_buf_size, sizeof(recv_buf_size))) { + perror("setsockopt(SO_RCVLOWAT)"); + exit(EXIT_FAILURE); + } + } + + fds.fd = fd; + fds.events = POLLIN | POLLRDNORM | POLLERR | + POLLRDHUP | POLLHUP; + + /* This 'poll()' will return once we receive last byte + * sent by client. + */ + if (poll(&fds, 1, -1) < 0) { + perror("poll"); + exit(EXIT_FAILURE); + } + + if (fds.revents & POLLERR) { + fprintf(stderr, "'poll()' error\n"); + exit(EXIT_FAILURE); + } + + if (fds.revents & (POLLIN | POLLRDNORM)) { + recv_buf(fd, buf, recv_buf_size, MSG_DONTWAIT, recv_buf_size); + } else { + /* These flags must be set, as there is at + * least 64KB of data ready to read. + */ + fprintf(stderr, "POLLIN | POLLRDNORM expected\n"); + exit(EXIT_FAILURE); + } + + free(buf); + close(fd); +} + +static void test_stream_cred_upd_on_low_rx_bytes(const struct test_opts *opts) +{ + test_stream_credit_update_test(opts, true); +} + +static void test_stream_cred_upd_on_set_rcvlowat(const struct test_opts *opts) +{ + test_stream_credit_update_test(opts, false); +} + static struct test_case test_cases[] = { { .name = "SOCK_STREAM connection reset", @@ -1342,6 +1507,16 @@ static struct test_case test_cases[] = { .run_client = test_double_bind_connect_client, .run_server = test_double_bind_connect_server, }, + { + .name = "SOCK_STREAM virtio credit update + SO_RCVLOWAT", + .run_client = test_stream_rcvlowat_def_cred_upd_client, + .run_server = test_stream_cred_upd_on_set_rcvlowat, + }, + { + .name = "SOCK_STREAM virtio credit update + low rx_bytes", + .run_client = test_stream_rcvlowat_def_cred_upd_client, + .run_server = test_stream_cred_upd_on_low_rx_bytes, + }, {}, };