Page MenuHomeFreeBSD
Files F110658650

D36944.diff
No OneTemporary
Actions

D36944.diff
View Options

This file is larger than 256 KB, so syntax highlighting was skipped.
diff --git a/contrib/ofed/libirdma/abi.h b/contrib/ofed/libirdma/abi.h
--- a/contrib/ofed/libirdma/abi.h
+++ b/contrib/ofed/libirdma/abi.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (C) 2019 - 2020 Intel Corporation
+ * Copyright (C) 2019 - 2022 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -145,6 +145,14 @@
__u16 rq_pages;
__u16 sq_pages;
+};
+struct irdma_urereg_mr {
+ struct ibv_rereg_mr ibv_cmd;
+ __u16 reg_type; /* enum irdma_memreg_type */
+ __u16 cq_pages;
+ __u16 rq_pages;
+ __u16 sq_pages;
+
};
struct irdma_ucreate_ah_resp {
struct ibv_create_ah_resp ibv_resp;
diff --git a/contrib/ofed/libirdma/i40iw_hw.h b/contrib/ofed/libirdma/i40iw_hw.h
--- a/contrib/ofed/libirdma/i40iw_hw.h
+++ b/contrib/ofed/libirdma/i40iw_hw.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2020 Intel Corporation
+ * Copyright (c) 2015 - 2022 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -50,11 +50,11 @@
I40IW_MAX_CQ_SIZE = 1048575,
I40IW_MAX_OUTBOUND_MSG_SIZE = 2147483647,
I40IW_MAX_INBOUND_MSG_SIZE = 2147483647,
+ I40IW_MIN_WQ_SIZE = 4 /* WQEs */,
};
#define I40IW_QP_WQE_MIN_SIZE 32
#define I40IW_QP_WQE_MAX_SIZE 128
-#define I40IW_QP_SW_MIN_WQSIZE 4
#define I40IW_MAX_RQ_WQE_SHIFT 2
#define I40IW_MAX_QUANTA_PER_WR 2
diff --git a/contrib/ofed/libirdma/irdma.h b/contrib/ofed/libirdma/irdma.h
--- a/contrib/ofed/libirdma/irdma.h
+++ b/contrib/ofed/libirdma/irdma.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2017 - 2021 Intel Corporation
+ * Copyright (c) 2017 - 2022 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -39,13 +39,15 @@
#define RDMA_BIT2(type, a) ((u##type) 1UL << a)
#define RDMA_MASK3(type, mask, shift) ((u##type) mask << shift)
#define MAKEMASK(m, s) ((m) << (s))
-#define IRDMA_WQEALLOC_WQE_DESC_INDEX_S 20
-#define IRDMA_WQEALLOC_WQE_DESC_INDEX_M (0xfff << IRDMA_WQEALLOC_WQE_DESC_INDEX_S)
+
+#define IRDMA_WQEALLOC_WQE_DESC_INDEX_S 20
+#define IRDMA_WQEALLOC_WQE_DESC_INDEX GENMASK(31, 20)
enum irdma_vers {
- IRDMA_GEN_RSVD,
- IRDMA_GEN_1,
- IRDMA_GEN_2,
+ IRDMA_GEN_RSVD = 0,
+ IRDMA_GEN_1 = 1,
+ IRDMA_GEN_2 = 2,
+ IRDMA_GEN_MAX = 2,
};
struct irdma_uk_attrs {
@@ -58,8 +60,7 @@
u32 min_hw_cq_size;
u32 max_hw_cq_size;
u16 max_hw_sq_chunk;
- u16 max_hw_wq_size;
- u16 min_sw_wq_size;
+ u16 min_hw_wq_size;
u8 hw_rev;
};
@@ -68,6 +69,7 @@
u64 max_hw_outbound_msg_size;
u64 max_hw_inbound_msg_size;
u64 max_mr_size;
+ u64 page_size_cap;
u32 min_hw_qp_id;
u32 min_hw_aeq_size;
u32 max_hw_aeq_size;
diff --git a/contrib/ofed/libirdma/irdma_defs.h b/contrib/ofed/libirdma/irdma_defs.h
--- a/contrib/ofed/libirdma/irdma_defs.h
+++ b/contrib/ofed/libirdma/irdma_defs.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2021 Intel Corporation
+ * Copyright (c) 2015 - 2022 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -75,7 +75,6 @@
#define IRDMA_CQE_QTYPE_RQ 0
#define IRDMA_CQE_QTYPE_SQ 1
-#define IRDMA_QP_SW_MIN_WQSIZE 8u /* in WRs*/
#define IRDMA_QP_WQE_MIN_SIZE 32
#define IRDMA_QP_WQE_MAX_SIZE 256
#define IRDMA_QP_WQE_MIN_QUANTA 1
@@ -85,9 +84,11 @@
#define IRDMA_SQ_RSVD 258
#define IRDMA_RQ_RSVD 1
-#define IRDMA_FEATURE_RTS_AE 1ULL
-#define IRDMA_FEATURE_CQ_RESIZE 2ULL
-#define IRDMA_FEATURE_RELAX_RQ_ORDER 4ULL
+#define IRDMA_FEATURE_RTS_AE BIT_ULL(0)
+#define IRDMA_FEATURE_CQ_RESIZE BIT_ULL(1)
+#define IRDMA_FEATURE_RELAX_RQ_ORDER BIT_ULL(2)
+#define IRDMA_FEATURE_64_BYTE_CQE BIT_ULL(5)
+
#define IRDMAQP_OP_RDMA_WRITE 0x00
#define IRDMAQP_OP_RDMA_READ 0x01
#define IRDMAQP_OP_RDMA_SEND 0x03
@@ -106,262 +107,198 @@
#define LS_32_1(val, bits) ((u32)((val) << (bits)))
#define RS_32_1(val, bits) ((u32)((val) >> (bits)))
#endif
-#define LS_64(val, field) (((u64)(val) << field ## _S) & (field ## _M))
-#define RS_64(val, field) ((u64)((val) & field ## _M) >> field ## _S)
-#define LS_32(val, field) (((val) << field ## _S) & (field ## _M))
-#define RS_32(val, field) (((val) & field ## _M) >> field ## _S)
+#ifndef GENMASK_ULL
+#define GENMASK_ULL(high, low) ((0xFFFFFFFFFFFFFFFFULL >> (64ULL - ((high) - (low) + 1ULL))) << (low))
+#endif /* GENMASK_ULL */
+#ifndef GENMASK
+#define GENMASK(high, low) ((0xFFFFFFFFUL >> (32UL - ((high) - (low) + 1UL))) << (low))
+#endif /* GENMASK */
+#ifndef FIELD_PREP
+#define FIELD_PREP(mask, val) (((u64)(val) << mask##_S) & (mask))
+#define FIELD_GET(mask, val) (((val) & mask) >> mask##_S)
+#endif /* FIELD_PREP */
#define IRDMA_CQPHC_QPCTX_S 0
-#define IRDMA_CQPHC_QPCTX_M \
- (0xffffffffffffffffULL << IRDMA_CQPHC_QPCTX_S)
-
-/* iWARP QP Doorbell shadow area */
+#define IRDMA_CQPHC_QPCTX GENMASK_ULL(63, 0)
#define IRDMA_QP_DBSA_HW_SQ_TAIL_S 0
-#define IRDMA_QP_DBSA_HW_SQ_TAIL_M \
- (0x7fffULL << IRDMA_QP_DBSA_HW_SQ_TAIL_S)
-
-/* Completion Queue Doorbell shadow area */
+#define IRDMA_QP_DBSA_HW_SQ_TAIL GENMASK_ULL(14, 0)
#define IRDMA_CQ_DBSA_CQEIDX_S 0
-#define IRDMA_CQ_DBSA_CQEIDX_M (0xfffffULL << IRDMA_CQ_DBSA_CQEIDX_S)
-
+#define IRDMA_CQ_DBSA_CQEIDX GENMASK_ULL(19, 0)
#define IRDMA_CQ_DBSA_SW_CQ_SELECT_S 0
-#define IRDMA_CQ_DBSA_SW_CQ_SELECT_M \
- (0x3fffULL << IRDMA_CQ_DBSA_SW_CQ_SELECT_S)
-
+#define IRDMA_CQ_DBSA_SW_CQ_SELECT GENMASK_ULL(13, 0)
#define IRDMA_CQ_DBSA_ARM_NEXT_S 14
-#define IRDMA_CQ_DBSA_ARM_NEXT_M BIT_ULL(IRDMA_CQ_DBSA_ARM_NEXT_S)
-
+#define IRDMA_CQ_DBSA_ARM_NEXT BIT_ULL(14)
#define IRDMA_CQ_DBSA_ARM_NEXT_SE_S 15
-#define IRDMA_CQ_DBSA_ARM_NEXT_SE_M BIT_ULL(IRDMA_CQ_DBSA_ARM_NEXT_SE_S)
-
+#define IRDMA_CQ_DBSA_ARM_NEXT_SE BIT_ULL(15)
#define IRDMA_CQ_DBSA_ARM_SEQ_NUM_S 16
-#define IRDMA_CQ_DBSA_ARM_SEQ_NUM_M \
- (0x3ULL << IRDMA_CQ_DBSA_ARM_SEQ_NUM_S)
+#define IRDMA_CQ_DBSA_ARM_SEQ_NUM GENMASK_ULL(17, 16)
/* CQP and iWARP Completion Queue */
#define IRDMA_CQ_QPCTX_S IRDMA_CQPHC_QPCTX_S
-#define IRDMA_CQ_QPCTX_M IRDMA_CQPHC_QPCTX_M
+#define IRDMA_CQ_QPCTX IRDMA_CQPHC_QPCTX
#define IRDMA_CQ_MINERR_S 0
-#define IRDMA_CQ_MINERR_M (0xffffULL << IRDMA_CQ_MINERR_S)
-
+#define IRDMA_CQ_MINERR GENMASK_ULL(15, 0)
#define IRDMA_CQ_MAJERR_S 16
-#define IRDMA_CQ_MAJERR_M (0xffffULL << IRDMA_CQ_MAJERR_S)
-
+#define IRDMA_CQ_MAJERR GENMASK_ULL(31, 16)
#define IRDMA_CQ_WQEIDX_S 32
-#define IRDMA_CQ_WQEIDX_M (0x7fffULL << IRDMA_CQ_WQEIDX_S)
-
+#define IRDMA_CQ_WQEIDX GENMASK_ULL(46, 32)
#define IRDMA_CQ_EXTCQE_S 50
-#define IRDMA_CQ_EXTCQE_M BIT_ULL(IRDMA_CQ_EXTCQE_S)
-
+#define IRDMA_CQ_EXTCQE BIT_ULL(50)
#define IRDMA_OOO_CMPL_S 54
-#define IRDMA_OOO_CMPL_M BIT_ULL(IRDMA_OOO_CMPL_S)
-
+#define IRDMA_OOO_CMPL BIT_ULL(54)
#define IRDMA_CQ_ERROR_S 55
-#define IRDMA_CQ_ERROR_M BIT_ULL(IRDMA_CQ_ERROR_S)
-
+#define IRDMA_CQ_ERROR BIT_ULL(55)
#define IRDMA_CQ_SQ_S 62
-#define IRDMA_CQ_SQ_M BIT_ULL(IRDMA_CQ_SQ_S)
+#define IRDMA_CQ_SQ BIT_ULL(62)
#define IRDMA_CQ_VALID_S 63
-#define IRDMA_CQ_VALID_M BIT_ULL(IRDMA_CQ_VALID_S)
-
-#define IRDMA_CQ_IMMVALID_S 62
-#define IRDMA_CQ_IMMVALID_M BIT_ULL(IRDMA_CQ_IMMVALID_S)
-
+#define IRDMA_CQ_VALID BIT_ULL(63)
+#define IRDMA_CQ_IMMVALID BIT_ULL(62)
#define IRDMA_CQ_UDSMACVALID_S 61
-#define IRDMA_CQ_UDSMACVALID_M BIT_ULL(IRDMA_CQ_UDSMACVALID_S)
-
+#define IRDMA_CQ_UDSMACVALID BIT_ULL(61)
#define IRDMA_CQ_UDVLANVALID_S 60
-#define IRDMA_CQ_UDVLANVALID_M BIT_ULL(IRDMA_CQ_UDVLANVALID_S)
-
+#define IRDMA_CQ_UDVLANVALID BIT_ULL(60)
#define IRDMA_CQ_UDSMAC_S 0
-#define IRDMA_CQ_UDSMAC_M (0xffffffffffffULL << IRDMA_CQ_UDSMAC_S)
-
+#define IRDMA_CQ_UDSMAC GENMASK_ULL(47, 0)
#define IRDMA_CQ_UDVLAN_S 48
-#define IRDMA_CQ_UDVLAN_M (0xffffULL << IRDMA_CQ_UDVLAN_S)
+#define IRDMA_CQ_UDVLAN GENMASK_ULL(63, 48)
#define IRDMA_CQ_IMMDATA_S 0
-#define IRDMA_CQ_IMMDATA_M (0xffffffffffffffffULL << IRDMA_CQ_IMMVALID_S)
-
+#define IRDMA_CQ_IMMVALID_S 62
+#define IRDMA_CQ_IMMDATA GENMASK_ULL(125, 62)
#define IRDMA_CQ_IMMDATALOW32_S 0
-#define IRDMA_CQ_IMMDATALOW32_M (0xffffffffULL << IRDMA_CQ_IMMDATALOW32_S)
-
+#define IRDMA_CQ_IMMDATALOW32 GENMASK_ULL(31, 0)
#define IRDMA_CQ_IMMDATAUP32_S 32
-#define IRDMA_CQ_IMMDATAUP32_M (0xffffffffULL << IRDMA_CQ_IMMDATAUP32_S)
-
+#define IRDMA_CQ_IMMDATAUP32 GENMASK_ULL(63, 32)
#define IRDMACQ_PAYLDLEN_S 0
-#define IRDMACQ_PAYLDLEN_M (0xffffffffULL << IRDMACQ_PAYLDLEN_S)
-
-#define IRDMACQ_TCPSEQNUMRTT_S 32
-#define IRDMACQ_TCPSEQNUMRTT_M (0xffffffffULL << IRDMACQ_TCPSEQNUMRTT_S)
-
+#define IRDMACQ_PAYLDLEN GENMASK_ULL(31, 0)
+#define IRDMACQ_TCPSQN_ROCEPSN_RTT_TS_S 32
+#define IRDMACQ_TCPSQN_ROCEPSN_RTT_TS GENMASK_ULL(63, 32)
#define IRDMACQ_INVSTAG_S 0
-#define IRDMACQ_INVSTAG_M (0xffffffffULL << IRDMACQ_INVSTAG_S)
-
+#define IRDMACQ_INVSTAG GENMASK_ULL(31, 0)
#define IRDMACQ_QPID_S 32
-#define IRDMACQ_QPID_M (0xffffffULL << IRDMACQ_QPID_S)
+#define IRDMACQ_QPID GENMASK_ULL(55, 32)
#define IRDMACQ_UDSRCQPN_S 0
-#define IRDMACQ_UDSRCQPN_M (0xffffffffULL << IRDMACQ_UDSRCQPN_S)
-
+#define IRDMACQ_UDSRCQPN GENMASK_ULL(31, 0)
#define IRDMACQ_PSHDROP_S 51
-#define IRDMACQ_PSHDROP_M BIT_ULL(IRDMACQ_PSHDROP_S)
-
+#define IRDMACQ_PSHDROP BIT_ULL(51)
#define IRDMACQ_STAG_S 53
-#define IRDMACQ_STAG_M BIT_ULL(IRDMACQ_STAG_S)
-
+#define IRDMACQ_STAG BIT_ULL(53)
#define IRDMACQ_IPV4_S 53
-#define IRDMACQ_IPV4_M BIT_ULL(IRDMACQ_IPV4_S)
-
+#define IRDMACQ_IPV4 BIT_ULL(53)
#define IRDMACQ_SOEVENT_S 54
-#define IRDMACQ_SOEVENT_M BIT_ULL(IRDMACQ_SOEVENT_S)
-
+#define IRDMACQ_SOEVENT BIT_ULL(54)
#define IRDMACQ_OP_S 56
-#define IRDMACQ_OP_M (0x3fULL << IRDMACQ_OP_S)
+#define IRDMACQ_OP GENMASK_ULL(61, 56)
/* Manage Push Page - MPP */
#define IRDMA_INVALID_PUSH_PAGE_INDEX_GEN_1 0xffff
#define IRDMA_INVALID_PUSH_PAGE_INDEX 0xffffffff
-/* iwarp QP SQ WQE common fields */
#define IRDMAQPSQ_OPCODE_S 32
-#define IRDMAQPSQ_OPCODE_M (0x3fULL << IRDMAQPSQ_OPCODE_S)
-
+#define IRDMAQPSQ_OPCODE GENMASK_ULL(37, 32)
#define IRDMAQPSQ_COPY_HOST_PBL_S 43
-#define IRDMAQPSQ_COPY_HOST_PBL_M BIT_ULL(IRDMAQPSQ_COPY_HOST_PBL_S)
-
+#define IRDMAQPSQ_COPY_HOST_PBL BIT_ULL(43)
#define IRDMAQPSQ_ADDFRAGCNT_S 38
-#define IRDMAQPSQ_ADDFRAGCNT_M (0xfULL << IRDMAQPSQ_ADDFRAGCNT_S)
-
+#define IRDMAQPSQ_ADDFRAGCNT GENMASK_ULL(41, 38)
#define IRDMAQPSQ_PUSHWQE_S 56
-#define IRDMAQPSQ_PUSHWQE_M BIT_ULL(IRDMAQPSQ_PUSHWQE_S)
-
+#define IRDMAQPSQ_PUSHWQE BIT_ULL(56)
#define IRDMAQPSQ_STREAMMODE_S 58
-#define IRDMAQPSQ_STREAMMODE_M BIT_ULL(IRDMAQPSQ_STREAMMODE_S)
-
+#define IRDMAQPSQ_STREAMMODE BIT_ULL(58)
#define IRDMAQPSQ_WAITFORRCVPDU_S 59
-#define IRDMAQPSQ_WAITFORRCVPDU_M BIT_ULL(IRDMAQPSQ_WAITFORRCVPDU_S)
-
+#define IRDMAQPSQ_WAITFORRCVPDU BIT_ULL(59)
#define IRDMAQPSQ_READFENCE_S 60
-#define IRDMAQPSQ_READFENCE_M BIT_ULL(IRDMAQPSQ_READFENCE_S)
-
+#define IRDMAQPSQ_READFENCE BIT_ULL(60)
#define IRDMAQPSQ_LOCALFENCE_S 61
-#define IRDMAQPSQ_LOCALFENCE_M BIT_ULL(IRDMAQPSQ_LOCALFENCE_S)
-
+#define IRDMAQPSQ_LOCALFENCE BIT_ULL(61)
#define IRDMAQPSQ_UDPHEADER_S 61
-#define IRDMAQPSQ_UDPHEADER_M BIT_ULL(IRDMAQPSQ_UDPHEADER_S)
-
+#define IRDMAQPSQ_UDPHEADER BIT_ULL(61)
#define IRDMAQPSQ_L4LEN_S 42
-#define IRDMAQPSQ_L4LEN_M ((u64)0xF << IRDMAQPSQ_L4LEN_S)
-
+#define IRDMAQPSQ_L4LEN GENMASK_ULL(45, 42)
#define IRDMAQPSQ_SIGCOMPL_S 62
-#define IRDMAQPSQ_SIGCOMPL_M BIT_ULL(IRDMAQPSQ_SIGCOMPL_S)
-
+#define IRDMAQPSQ_SIGCOMPL BIT_ULL(62)
#define IRDMAQPSQ_VALID_S 63
-#define IRDMAQPSQ_VALID_M BIT_ULL(IRDMAQPSQ_VALID_S)
+#define IRDMAQPSQ_VALID BIT_ULL(63)
#define IRDMAQPSQ_FRAG_TO_S IRDMA_CQPHC_QPCTX_S
-#define IRDMAQPSQ_FRAG_TO_M IRDMA_CQPHC_QPCTX_M
-
+#define IRDMAQPSQ_FRAG_TO IRDMA_CQPHC_QPCTX
#define IRDMAQPSQ_FRAG_VALID_S 63
-#define IRDMAQPSQ_FRAG_VALID_M BIT_ULL(IRDMAQPSQ_FRAG_VALID_S)
-
+#define IRDMAQPSQ_FRAG_VALID BIT_ULL(63)
#define IRDMAQPSQ_FRAG_LEN_S 32
-#define IRDMAQPSQ_FRAG_LEN_M (0x7fffffffULL << IRDMAQPSQ_FRAG_LEN_S)
-
+#define IRDMAQPSQ_FRAG_LEN GENMASK_ULL(62, 32)
#define IRDMAQPSQ_FRAG_STAG_S 0
-#define IRDMAQPSQ_FRAG_STAG_M (0xffffffffULL << IRDMAQPSQ_FRAG_STAG_S)
-
+#define IRDMAQPSQ_FRAG_STAG GENMASK_ULL(31, 0)
#define IRDMAQPSQ_GEN1_FRAG_LEN_S 0
-#define IRDMAQPSQ_GEN1_FRAG_LEN_M (0xffffffffULL << IRDMAQPSQ_GEN1_FRAG_LEN_S)
-
+#define IRDMAQPSQ_GEN1_FRAG_LEN GENMASK_ULL(31, 0)
#define IRDMAQPSQ_GEN1_FRAG_STAG_S 32
-#define IRDMAQPSQ_GEN1_FRAG_STAG_M (0xffffffffULL << IRDMAQPSQ_GEN1_FRAG_STAG_S)
-
+#define IRDMAQPSQ_GEN1_FRAG_STAG GENMASK_ULL(63, 32)
#define IRDMAQPSQ_REMSTAGINV_S 0
-#define IRDMAQPSQ_REMSTAGINV_M (0xffffffffULL << IRDMAQPSQ_REMSTAGINV_S)
-
+#define IRDMAQPSQ_REMSTAGINV GENMASK_ULL(31, 0)
#define IRDMAQPSQ_DESTQKEY_S 0
-#define IRDMAQPSQ_DESTQKEY_M (0xffffffffULL << IRDMAQPSQ_DESTQKEY_S)
-
+#define IRDMAQPSQ_DESTQKEY GENMASK_ULL(31, 0)
#define IRDMAQPSQ_DESTQPN_S 32
-#define IRDMAQPSQ_DESTQPN_M (0x00ffffffULL << IRDMAQPSQ_DESTQPN_S)
-
+#define IRDMAQPSQ_DESTQPN GENMASK_ULL(55, 32)
#define IRDMAQPSQ_AHID_S 0
-#define IRDMAQPSQ_AHID_M (0x0001ffffULL << IRDMAQPSQ_AHID_S)
-
+#define IRDMAQPSQ_AHID GENMASK_ULL(16, 0)
#define IRDMAQPSQ_INLINEDATAFLAG_S 57
-#define IRDMAQPSQ_INLINEDATAFLAG_M BIT_ULL(IRDMAQPSQ_INLINEDATAFLAG_S)
+#define IRDMAQPSQ_INLINEDATAFLAG BIT_ULL(57)
#define IRDMA_INLINE_VALID_S 7
-
#define IRDMAQPSQ_INLINEDATALEN_S 48
-#define IRDMAQPSQ_INLINEDATALEN_M \
- (0xffULL << IRDMAQPSQ_INLINEDATALEN_S)
+#define IRDMAQPSQ_INLINEDATALEN GENMASK_ULL(55, 48)
#define IRDMAQPSQ_IMMDATAFLAG_S 47
-#define IRDMAQPSQ_IMMDATAFLAG_M \
- BIT_ULL(IRDMAQPSQ_IMMDATAFLAG_S)
+#define IRDMAQPSQ_IMMDATAFLAG BIT_ULL(47)
#define IRDMAQPSQ_REPORTRTT_S 46
-#define IRDMAQPSQ_REPORTRTT_M \
- BIT_ULL(IRDMAQPSQ_REPORTRTT_S)
+#define IRDMAQPSQ_REPORTRTT BIT_ULL(46)
#define IRDMAQPSQ_IMMDATA_S 0
-#define IRDMAQPSQ_IMMDATA_M \
- (0xffffffffffffffffULL << IRDMAQPSQ_IMMDATA_S)
-
-/* rdma write */
+#define IRDMAQPSQ_IMMDATA GENMASK_ULL(63, 0)
#define IRDMAQPSQ_REMSTAG_S 0
-#define IRDMAQPSQ_REMSTAG_M (0xffffffffULL << IRDMAQPSQ_REMSTAG_S)
+#define IRDMAQPSQ_REMSTAG GENMASK_ULL(31, 0)
#define IRDMAQPSQ_REMTO_S IRDMA_CQPHC_QPCTX_S
-#define IRDMAQPSQ_REMTO_M IRDMA_CQPHC_QPCTX_M
+#define IRDMAQPSQ_REMTO IRDMA_CQPHC_QPCTX
-/* memory window */
#define IRDMAQPSQ_STAGRIGHTS_S 48
-#define IRDMAQPSQ_STAGRIGHTS_M (0x1fULL << IRDMAQPSQ_STAGRIGHTS_S)
-
+#define IRDMAQPSQ_STAGRIGHTS GENMASK_ULL(52, 48)
#define IRDMAQPSQ_VABASEDTO_S 53
-#define IRDMAQPSQ_VABASEDTO_M BIT_ULL(IRDMAQPSQ_VABASEDTO_S)
-
+#define IRDMAQPSQ_VABASEDTO BIT_ULL(53)
#define IRDMAQPSQ_MEMWINDOWTYPE_S 54
-#define IRDMAQPSQ_MEMWINDOWTYPE_M BIT_ULL(IRDMAQPSQ_MEMWINDOWTYPE_S)
+#define IRDMAQPSQ_MEMWINDOWTYPE BIT_ULL(54)
#define IRDMAQPSQ_MWLEN_S IRDMA_CQPHC_QPCTX_S
-#define IRDMAQPSQ_MWLEN_M IRDMA_CQPHC_QPCTX_M
-
+#define IRDMAQPSQ_MWLEN IRDMA_CQPHC_QPCTX
#define IRDMAQPSQ_PARENTMRSTAG_S 32
-#define IRDMAQPSQ_PARENTMRSTAG_M \
- (0xffffffffULL << IRDMAQPSQ_PARENTMRSTAG_S)
-
+#define IRDMAQPSQ_PARENTMRSTAG GENMASK_ULL(63, 32)
#define IRDMAQPSQ_MWSTAG_S 0
-#define IRDMAQPSQ_MWSTAG_M (0xffffffffULL << IRDMAQPSQ_MWSTAG_S)
+#define IRDMAQPSQ_MWSTAG GENMASK_ULL(31, 0)
#define IRDMAQPSQ_BASEVA_TO_FBO_S IRDMA_CQPHC_QPCTX_S
-#define IRDMAQPSQ_BASEVA_TO_FBO_M IRDMA_CQPHC_QPCTX_M
+#define IRDMAQPSQ_BASEVA_TO_FBO IRDMA_CQPHC_QPCTX
-/* Local Invalidate */
#define IRDMAQPSQ_LOCSTAG_S 0
-#define IRDMAQPSQ_LOCSTAG_M (0xffffffffULL << IRDMAQPSQ_LOCSTAG_S)
+#define IRDMAQPSQ_LOCSTAG GENMASK_ULL(31, 0)
/* iwarp QP RQ WQE common fields */
#define IRDMAQPRQ_ADDFRAGCNT_S IRDMAQPSQ_ADDFRAGCNT_S
-#define IRDMAQPRQ_ADDFRAGCNT_M IRDMAQPSQ_ADDFRAGCNT_M
+#define IRDMAQPRQ_ADDFRAGCNT IRDMAQPSQ_ADDFRAGCNT
#define IRDMAQPRQ_VALID_S IRDMAQPSQ_VALID_S
-#define IRDMAQPRQ_VALID_M IRDMAQPSQ_VALID_M
+#define IRDMAQPRQ_VALID IRDMAQPSQ_VALID
#define IRDMAQPRQ_COMPLCTX_S IRDMA_CQPHC_QPCTX_S
-#define IRDMAQPRQ_COMPLCTX_M IRDMA_CQPHC_QPCTX_M
+#define IRDMAQPRQ_COMPLCTX IRDMA_CQPHC_QPCTX
#define IRDMAQPRQ_FRAG_LEN_S IRDMAQPSQ_FRAG_LEN_S
-#define IRDMAQPRQ_FRAG_LEN_M IRDMAQPSQ_FRAG_LEN_M
+#define IRDMAQPRQ_FRAG_LEN IRDMAQPSQ_FRAG_LEN
#define IRDMAQPRQ_STAG_S IRDMAQPSQ_FRAG_STAG_S
-#define IRDMAQPRQ_STAG_M IRDMAQPSQ_FRAG_STAG_M
+#define IRDMAQPRQ_STAG IRDMAQPSQ_FRAG_STAG
#define IRDMAQPRQ_TO_S IRDMAQPSQ_FRAG_TO_S
-#define IRDMAQPRQ_TO_M IRDMAQPSQ_FRAG_TO_M
+#define IRDMAQPRQ_TO IRDMAQPSQ_FRAG_TO
#define IRDMAPFINT_OICR_HMC_ERR_M BIT(26)
#define IRDMAPFINT_OICR_PE_PUSH_M BIT(27)
@@ -500,6 +437,12 @@
IRDMA_RING_MOVE_HEAD(_ring, _retcode); \
}
+enum irdma_protocol_used {
+ IRDMA_ANY_PROTOCOL = 0,
+ IRDMA_IWARP_PROTOCOL_ONLY = 1,
+ IRDMA_ROCE_PROTOCOL_ONLY = 2,
+};
+
enum irdma_qp_wqe_size {
IRDMA_WQE_SIZE_32 = 32,
IRDMA_WQE_SIZE_64 = 64,
diff --git a/contrib/ofed/libirdma/irdma_uk.c b/contrib/ofed/libirdma/irdma_uk.c
--- a/contrib/ofed/libirdma/irdma_uk.c
+++ b/contrib/ofed/libirdma/irdma_uk.c
@@ -51,15 +51,15 @@
{
if (sge) {
set_64bit_val(wqe, offset,
- LS_64(sge->tag_off, IRDMAQPSQ_FRAG_TO));
+ FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->tag_off));
set_64bit_val(wqe, offset + IRDMA_BYTE_8,
- LS_64(valid, IRDMAQPSQ_VALID) |
- LS_64(sge->len, IRDMAQPSQ_FRAG_LEN) |
- LS_64(sge->stag, IRDMAQPSQ_FRAG_STAG));
+ FIELD_PREP(IRDMAQPSQ_VALID, valid) |
+ FIELD_PREP(IRDMAQPSQ_FRAG_LEN, sge->len) |
+ FIELD_PREP(IRDMAQPSQ_FRAG_STAG, sge->stag));
} else {
set_64bit_val(wqe, offset, 0);
set_64bit_val(wqe, offset + IRDMA_BYTE_8,
- LS_64(valid, IRDMAQPSQ_VALID));
+ FIELD_PREP(IRDMAQPSQ_VALID, valid));
}
}
@@ -76,16 +76,26 @@
{
if (sge) {
set_64bit_val(wqe, offset,
- LS_64(sge->tag_off, IRDMAQPSQ_FRAG_TO));
+ FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->tag_off));
set_64bit_val(wqe, offset + IRDMA_BYTE_8,
- LS_64(sge->len, IRDMAQPSQ_GEN1_FRAG_LEN) |
- LS_64(sge->stag, IRDMAQPSQ_GEN1_FRAG_STAG));
+ FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, sge->len) |
+ FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, sge->stag));
} else {
set_64bit_val(wqe, offset, 0);
set_64bit_val(wqe, offset + IRDMA_BYTE_8, 0);
}
}
+/**
+ * irdma_nop_hdr - Format header section of noop WQE
+ * @qp: hw qp ptr
+ */
+static inline u64 irdma_nop_hdr(struct irdma_qp_uk *qp){
+ return FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) |
+ FIELD_PREP(IRDMAQPSQ_SIGCOMPL, false) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
+}
+
/**
* irdma_nop_1 - insert a NOP wqe
* @qp: hw qp ptr
@@ -93,10 +103,8 @@
static int
irdma_nop_1(struct irdma_qp_uk *qp)
{
- u64 hdr;
__le64 *wqe;
u32 wqe_idx;
- bool signaled = false;
if (!qp->sq_ring.head)
return EINVAL;
@@ -110,14 +118,10 @@
set_64bit_val(wqe, IRDMA_BYTE_8, 0);
set_64bit_val(wqe, IRDMA_BYTE_16, 0);
- hdr = LS_64(IRDMAQP_OP_NOP, IRDMAQPSQ_OPCODE) |
- LS_64(signaled, IRDMAQPSQ_SIGCOMPL) |
- LS_64(qp->swqe_polarity, IRDMAQPSQ_VALID);
-
/* make sure WQE is written before valid bit is set */
udma_to_device_barrier();
- set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
+ set_64bit_val(wqe, IRDMA_BYTE_24, irdma_nop_hdr(qp));
return 0;
}
@@ -160,7 +164,7 @@
/* read the doorbell shadow area */
get_64bit_val(qp->shadow_area, IRDMA_BYTE_0, &temp);
- hw_sq_tail = (u32)RS_64(temp, IRDMA_QP_DBSA_HW_SQ_TAIL);
+ hw_sq_tail = (u32)FIELD_GET(IRDMA_QP_DBSA_HW_SQ_TAIL, temp);
sw_sq_head = IRDMA_RING_CURRENT_HEAD(qp->sq_ring);
if (sw_sq_head != qp->initial_ring.head) {
if (qp->push_dropped) {
@@ -191,7 +195,7 @@
irdma_qp_ring_push_db(struct irdma_qp_uk *qp, u32 wqe_idx)
{
set_32bit_val(qp->push_db, 0,
- LS_32(wqe_idx >> 3, IRDMA_WQEALLOC_WQE_DESC_INDEX) | qp->qp_id);
+ FIELD_PREP(IRDMA_WQEALLOC_WQE_DESC_INDEX, wqe_idx >> 3) | qp->qp_id);
qp->initial_ring.head = qp->sq_ring.head;
qp->push_mode = true;
qp->push_dropped = false;
@@ -220,31 +224,32 @@
* irdma_qp_get_next_send_wqe - pad with NOP if needed, return where next WR should go
* @qp: hw qp ptr
* @wqe_idx: return wqe index
- * @quanta: size of WR in quanta
+ * @quanta: (in/out) ptr to size of WR in quanta. Modified in case pad is needed
* @total_size: size of WR in bytes
* @info: info on WR
*/
__le64 *
irdma_qp_get_next_send_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx,
- u16 quanta, u32 total_size,
+ u16 *quanta, u32 total_size,
struct irdma_post_sq_info *info)
{
__le64 *wqe;
__le64 *wqe_0 = NULL;
u32 nop_wqe_idx;
- u16 avail_quanta;
+ u16 avail_quanta, wqe_quanta = *quanta;
u16 i;
avail_quanta = qp->uk_attrs->max_hw_sq_chunk -
(IRDMA_RING_CURRENT_HEAD(qp->sq_ring) %
qp->uk_attrs->max_hw_sq_chunk);
- if (quanta <= avail_quanta) {
+
+ if (*quanta <= avail_quanta) {
/* WR fits in current chunk */
- if (quanta > IRDMA_SQ_RING_FREE_QUANTA(qp->sq_ring))
+ if (*quanta > IRDMA_SQ_RING_FREE_QUANTA(qp->sq_ring))
return NULL;
} else {
/* Need to pad with NOP */
- if (quanta + avail_quanta >
+ if (*quanta + avail_quanta >
IRDMA_SQ_RING_FREE_QUANTA(qp->sq_ring))
return NULL;
@@ -262,17 +267,19 @@
if (!*wqe_idx)
qp->swqe_polarity = !qp->swqe_polarity;
- IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, quanta);
+ IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, *quanta);
+
+ irdma_clr_wqes(qp, *wqe_idx);
wqe = qp->sq_base[*wqe_idx].elem;
- if (qp->uk_attrs->hw_rev == IRDMA_GEN_1 && quanta == 1 &&
+ if (qp->uk_attrs->hw_rev == IRDMA_GEN_1 && wqe_quanta == 1 &&
(IRDMA_RING_CURRENT_HEAD(qp->sq_ring) & 1)) {
wqe_0 = qp->sq_base[IRDMA_RING_CURRENT_HEAD(qp->sq_ring)].elem;
- wqe_0[3] = htole64(LS_64(!qp->swqe_polarity, IRDMAQPSQ_VALID));
+ wqe_0[3] = htole64(FIELD_PREP(IRDMAQPSQ_VALID, !qp->swqe_polarity));
}
qp->sq_wrtrk_array[*wqe_idx].wrid = info->wr_id;
qp->sq_wrtrk_array[*wqe_idx].wr_len = total_size;
- qp->sq_wrtrk_array[*wqe_idx].quanta = quanta;
+ qp->sq_wrtrk_array[*wqe_idx].quanta = wqe_quanta;
qp->sq_wrtrk_array[*wqe_idx].signaled = info->signaled;
return wqe;
@@ -344,20 +351,17 @@
if (ret_code)
return ret_code;
- wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
- info);
+ wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, info);
if (!wqe)
return ENOSPC;
- irdma_clr_wqes(qp, wqe_idx);
-
qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled;
set_64bit_val(wqe, IRDMA_BYTE_16,
- LS_64(op_info->rem_addr.tag_off, IRDMAQPSQ_FRAG_TO));
+ FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off));
if (info->imm_data_valid) {
set_64bit_val(wqe, IRDMA_BYTE_0,
- LS_64(info->imm_data, IRDMAQPSQ_IMMDATA));
+ FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data));
i = 0;
} else {
qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0,
@@ -382,28 +386,24 @@
++addl_frag_cnt;
}
- if (!op_info->rem_addr.stag && !total_size)
- op_info->rem_addr.stag = 0x1234;
- hdr = LS_64(op_info->rem_addr.stag, IRDMAQPSQ_REMSTAG) |
- LS_64(info->op_type, IRDMAQPSQ_OPCODE) |
- LS_64((info->imm_data_valid ? 1 : 0), IRDMAQPSQ_IMMDATAFLAG) |
- LS_64((info->report_rtt ? 1 : 0), IRDMAQPSQ_REPORTRTT) |
- LS_64(addl_frag_cnt, IRDMAQPSQ_ADDFRAGCNT) |
- LS_64((info->push_wqe ? 1 : 0), IRDMAQPSQ_PUSHWQE) |
- LS_64(read_fence, IRDMAQPSQ_READFENCE) |
- LS_64(info->local_fence, IRDMAQPSQ_LOCALFENCE) |
- LS_64(info->signaled, IRDMAQPSQ_SIGCOMPL) |
- LS_64(qp->swqe_polarity, IRDMAQPSQ_VALID);
+ hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) |
+ FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) |
+ FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, info->imm_data_valid) |
+ FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt) |
+ FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) |
+ FIELD_PREP(IRDMAQPSQ_PUSHWQE, info->push_wqe) |
+ FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) |
+ FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) |
+ FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
- if (info->push_wqe) {
+ if (info->push_wqe)
irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);
- } else {
- if (post_sq)
- irdma_uk_qp_post_wr(qp);
- }
+ else if (post_sq)
+ irdma_uk_qp_post_wr(qp);
return 0;
}
@@ -443,8 +443,7 @@
if (ret_code)
return ret_code;
- wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
- info);
+ wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, info);
if (!wqe)
return ENOSPC;
@@ -453,8 +452,6 @@
qp->ord_cnt = 0;
}
- irdma_clr_wqes(qp, wqe_idx);
-
qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled;
addl_frag_cnt = op_info->num_lo_sges > 1 ?
(op_info->num_lo_sges - 1) : 0;
@@ -478,28 +475,26 @@
++addl_frag_cnt;
}
set_64bit_val(wqe, IRDMA_BYTE_16,
- LS_64(op_info->rem_addr.tag_off, IRDMAQPSQ_FRAG_TO));
- hdr = LS_64(op_info->rem_addr.stag, IRDMAQPSQ_REMSTAG) |
- LS_64((info->report_rtt ? 1 : 0), IRDMAQPSQ_REPORTRTT) |
- LS_64(addl_frag_cnt, IRDMAQPSQ_ADDFRAGCNT) |
- LS_64((inv_stag ? IRDMAQP_OP_RDMA_READ_LOC_INV : IRDMAQP_OP_RDMA_READ),
- IRDMAQPSQ_OPCODE) |
- LS_64((info->push_wqe ? 1 : 0), IRDMAQPSQ_PUSHWQE) |
- LS_64(info->read_fence || qp->force_fence || ord_fence ? 1 : 0,
- IRDMAQPSQ_READFENCE) |
- LS_64(local_fence, IRDMAQPSQ_LOCALFENCE) |
- LS_64(info->signaled, IRDMAQPSQ_SIGCOMPL) |
- LS_64(qp->swqe_polarity, IRDMAQPSQ_VALID);
+ FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off));
+ hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) |
+ FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) |
+ FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) |
+ FIELD_PREP(IRDMAQPSQ_OPCODE,
+ (inv_stag ? IRDMAQP_OP_RDMA_READ_LOC_INV : IRDMAQP_OP_RDMA_READ)) |
+ FIELD_PREP(IRDMAQPSQ_PUSHWQE, info->push_wqe) |
+ FIELD_PREP(IRDMAQPSQ_READFENCE,
+ info->read_fence || ord_fence ? 1 : 0) |
+ FIELD_PREP(IRDMAQPSQ_LOCALFENCE, local_fence) |
+ FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
- if (info->push_wqe) {
+ if (info->push_wqe)
irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);
- } else {
- if (post_sq)
- irdma_uk_qp_post_wr(qp);
- }
+ else if (post_sq)
+ irdma_uk_qp_post_wr(qp);
return 0;
}
@@ -540,21 +535,19 @@
if (ret_code)
return ret_code;
- wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
- info);
+ wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, info);
if (!wqe)
return ENOSPC;
- irdma_clr_wqes(qp, wqe_idx);
-
read_fence |= info->read_fence;
addl_frag_cnt = frag_cnt > 1 ? (frag_cnt - 1) : 0;
if (info->imm_data_valid) {
set_64bit_val(wqe, IRDMA_BYTE_0,
- LS_64(info->imm_data, IRDMAQPSQ_IMMDATA));
+ FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data));
i = 0;
} else {
- qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0, op_info->sg_list,
+ qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0,
+ frag_cnt ? op_info->sg_list : NULL,
qp->swqe_polarity);
i = 1;
}
@@ -575,31 +568,30 @@
}
set_64bit_val(wqe, IRDMA_BYTE_16,
- LS_64(op_info->qkey, IRDMAQPSQ_DESTQKEY) |
- LS_64(op_info->dest_qp, IRDMAQPSQ_DESTQPN));
- hdr = LS_64(info->stag_to_inv, IRDMAQPSQ_REMSTAG) |
- LS_64(op_info->ah_id, IRDMAQPSQ_AHID) |
- LS_64((info->imm_data_valid ? 1 : 0), IRDMAQPSQ_IMMDATAFLAG) |
- LS_64((info->report_rtt ? 1 : 0), IRDMAQPSQ_REPORTRTT) |
- LS_64(info->op_type, IRDMAQPSQ_OPCODE) |
- LS_64(addl_frag_cnt, IRDMAQPSQ_ADDFRAGCNT) |
- LS_64((info->push_wqe ? 1 : 0), IRDMAQPSQ_PUSHWQE) |
- LS_64(read_fence, IRDMAQPSQ_READFENCE) |
- LS_64(info->local_fence, IRDMAQPSQ_LOCALFENCE) |
- LS_64(info->signaled, IRDMAQPSQ_SIGCOMPL) |
- LS_64(info->udp_hdr, IRDMAQPSQ_UDPHEADER) |
- LS_64(info->l4len, IRDMAQPSQ_L4LEN) |
- LS_64(qp->swqe_polarity, IRDMAQPSQ_VALID);
+ FIELD_PREP(IRDMAQPSQ_DESTQKEY, op_info->qkey) |
+ FIELD_PREP(IRDMAQPSQ_DESTQPN, op_info->dest_qp));
+ hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, info->stag_to_inv) |
+ FIELD_PREP(IRDMAQPSQ_AHID, op_info->ah_id) |
+ FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG,
+ (info->imm_data_valid ? 1 : 0)) |
+ FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) |
+ FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) |
+ FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) |
+ FIELD_PREP(IRDMAQPSQ_PUSHWQE, info->push_wqe) |
+ FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) |
+ FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) |
+ FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
+ FIELD_PREP(IRDMAQPSQ_UDPHEADER, info->udp_hdr) |
+ FIELD_PREP(IRDMAQPSQ_L4LEN, info->l4len) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
- if (info->push_wqe) {
+ if (info->push_wqe)
irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);
- } else {
- if (post_sq)
- irdma_uk_qp_post_wr(qp);
- }
+ else if (post_sq)
+ irdma_uk_qp_post_wr(qp);
return 0;
}
@@ -615,29 +607,45 @@
{
set_64bit_val(wqe, IRDMA_BYTE_0, (uintptr_t)op_info->va);
set_64bit_val(wqe, IRDMA_BYTE_8,
- LS_64(op_info->mw_stag, IRDMAQPSQ_PARENTMRSTAG) |
- LS_64(op_info->mr_stag, IRDMAQPSQ_MWSTAG));
+ FIELD_PREP(IRDMAQPSQ_PARENTMRSTAG, op_info->mw_stag) |
+ FIELD_PREP(IRDMAQPSQ_MWSTAG, op_info->mr_stag));
set_64bit_val(wqe, IRDMA_BYTE_16, op_info->bind_len);
}
/**
* irdma_copy_inline_data_gen_1 - Copy inline data to wqe
- * @dest: pointer to wqe
- * @src: pointer to inline data
- * @len: length of inline data to copy
+ * @wqe: pointer to wqe
+ * @sge_list: table of pointers to inline data
+ * @num_sges: Total inline data length
* @polarity: compatibility parameter
*/
static void
-irdma_copy_inline_data_gen_1(u8 *dest, u8 *src, u32 len,
- u8 polarity)
+irdma_copy_inline_data_gen_1(u8 *wqe, struct irdma_sge *sge_list,
+ u32 num_sges, u8 polarity)
{
- if (len <= IRDMA_BYTE_16) {
- irdma_memcpy(dest, src, len);
- } else {
- irdma_memcpy(dest, src, IRDMA_BYTE_16);
- src += IRDMA_BYTE_16;
- dest = dest + IRDMA_BYTE_32;
- irdma_memcpy(dest, src, len - IRDMA_BYTE_16);
+ u32 quanta_bytes_remaining = 16;
+ u32 i;
+
+ for (i = 0; i < num_sges; i++) {
+ u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].tag_off;
+ u32 sge_len = sge_list[i].len;
+
+ while (sge_len) {
+ u32 bytes_copied;
+
+ bytes_copied = min(sge_len, quanta_bytes_remaining);
+ irdma_memcpy(wqe, cur_sge, bytes_copied);
+ wqe += bytes_copied;
+ cur_sge += bytes_copied;
+ quanta_bytes_remaining -= bytes_copied;
+ sge_len -= bytes_copied;
+
+ if (!quanta_bytes_remaining) {
+ /* Remaining inline bytes reside after the hdr */
+ wqe += 16;
+ quanta_bytes_remaining = 32;
+ }
+ }
}
}
@@ -662,43 +670,59 @@
{
set_64bit_val(wqe, IRDMA_BYTE_0, (uintptr_t)op_info->va);
set_64bit_val(wqe, IRDMA_BYTE_8,
- LS_64(op_info->mr_stag, IRDMAQPSQ_PARENTMRSTAG) |
- LS_64(op_info->mw_stag, IRDMAQPSQ_MWSTAG));
+ FIELD_PREP(IRDMAQPSQ_PARENTMRSTAG, op_info->mr_stag) |
+ FIELD_PREP(IRDMAQPSQ_MWSTAG, op_info->mw_stag));
set_64bit_val(wqe, IRDMA_BYTE_16, op_info->bind_len);
}
/**
* irdma_copy_inline_data - Copy inline data to wqe
- * @dest: pointer to wqe
- * @src: pointer to inline data
- * @len: length of inline data to copy
+ * @wqe: pointer to wqe
+ * @sge_list: table of pointers to inline data
+ * @num_sges: number of SGE's
* @polarity: polarity of wqe valid bit
*/
static void
-irdma_copy_inline_data(u8 *dest, u8 *src, u32 len, u8 polarity)
+irdma_copy_inline_data(u8 *wqe, struct irdma_sge *sge_list, u32 num_sges,
+ u8 polarity)
{
u8 inline_valid = polarity << IRDMA_INLINE_VALID_S;
- u32 copy_size;
-
- dest += IRDMA_BYTE_8;
- if (len <= IRDMA_BYTE_8) {
- irdma_memcpy(dest, src, len);
- return;
- }
-
- *((u64 *)dest) = *((u64 *)src);
- len -= IRDMA_BYTE_8;
- src += IRDMA_BYTE_8;
- dest += IRDMA_BYTE_24; /* point to additional 32 byte quanta */
-
- while (len) {
- copy_size = len < 31 ? len : 31;
- irdma_memcpy(dest, src, copy_size);
- *(dest + 31) = inline_valid;
- len -= copy_size;
- dest += IRDMA_BYTE_32;
- src += copy_size;
+ u32 quanta_bytes_remaining = 8;
+ u32 i;
+ bool first_quanta = true;
+
+ wqe += 8;
+
+ for (i = 0; i < num_sges; i++) {
+ u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].tag_off;
+ u32 sge_len = sge_list[i].len;
+
+ while (sge_len) {
+ u32 bytes_copied;
+
+ bytes_copied = min(sge_len, quanta_bytes_remaining);
+ irdma_memcpy(wqe, cur_sge, bytes_copied);
+ wqe += bytes_copied;
+ cur_sge += bytes_copied;
+ quanta_bytes_remaining -= bytes_copied;
+ sge_len -= bytes_copied;
+
+ if (!quanta_bytes_remaining) {
+ quanta_bytes_remaining = 31;
+
+ /* Remaining inline bytes reside after the hdr */
+ if (first_quanta) {
+ first_quanta = false;
+ wqe += 16;
+ } else {
+ *wqe = inline_valid;
+ wqe++;
+ }
+ }
+ }
}
+ if (!first_quanta && quanta_bytes_remaining < 31)
+ *(wqe + quanta_bytes_remaining) = inline_valid;
}
/**
@@ -737,59 +761,62 @@
struct irdma_post_sq_info *info, bool post_sq)
{
__le64 *wqe;
- struct irdma_inline_rdma_write *op_info;
+ struct irdma_rdma_write *op_info;
u64 hdr = 0;
u32 wqe_idx;
bool read_fence = false;
u16 quanta;
+ u32 i, total_size = 0;
info->push_wqe = qp->push_db ? true : false;
- op_info = &info->op.inline_rdma_write;
+ op_info = &info->op.rdma_write;
+
+ if (unlikely(qp->max_sq_frag_cnt < op_info->num_lo_sges))
+ return EINVAL;
+
+ for (i = 0; i < op_info->num_lo_sges; i++)
+ total_size += op_info->lo_sg_list[i].len;
- if (op_info->len > qp->max_inline_data)
+ if (unlikely(total_size > qp->max_inline_data))
return EINVAL;
- quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(op_info->len);
- wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, op_info->len,
- info);
+ quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(total_size);
+ wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, info);
if (!wqe)
return ENOSPC;
- irdma_clr_wqes(qp, wqe_idx);
-
qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled;
read_fence |= info->read_fence;
set_64bit_val(wqe, IRDMA_BYTE_16,
- LS_64(op_info->rem_addr.tag_off, IRDMAQPSQ_FRAG_TO));
-
- hdr = LS_64(op_info->rem_addr.stag, IRDMAQPSQ_REMSTAG) |
- LS_64(info->op_type, IRDMAQPSQ_OPCODE) |
- LS_64(op_info->len, IRDMAQPSQ_INLINEDATALEN) |
- LS_64(info->report_rtt ? 1 : 0, IRDMAQPSQ_REPORTRTT) |
- LS_64(1, IRDMAQPSQ_INLINEDATAFLAG) |
- LS_64(info->imm_data_valid ? 1 : 0, IRDMAQPSQ_IMMDATAFLAG) |
- LS_64(info->push_wqe ? 1 : 0, IRDMAQPSQ_PUSHWQE) |
- LS_64(read_fence, IRDMAQPSQ_READFENCE) |
- LS_64(info->local_fence, IRDMAQPSQ_LOCALFENCE) |
- LS_64(info->signaled, IRDMAQPSQ_SIGCOMPL) |
- LS_64(qp->swqe_polarity, IRDMAQPSQ_VALID);
+ FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off));
+
+ hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) |
+ FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) |
+ FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, total_size) |
+ FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt ? 1 : 0) |
+ FIELD_PREP(IRDMAQPSQ_INLINEDATAFLAG, 1) |
+ FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, info->imm_data_valid ? 1 : 0) |
+ FIELD_PREP(IRDMAQPSQ_PUSHWQE, info->push_wqe ? 1 : 0) |
+ FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) |
+ FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) |
+ FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
if (info->imm_data_valid)
set_64bit_val(wqe, IRDMA_BYTE_0,
- LS_64(info->imm_data, IRDMAQPSQ_IMMDATA));
+ FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data));
+
+ qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->lo_sg_list,
+ op_info->num_lo_sges, qp->swqe_polarity);
- qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->data, op_info->len,
- qp->swqe_polarity);
udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
- if (info->push_wqe) {
+ if (info->push_wqe)
irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);
- } else {
- if (post_sq)
- irdma_uk_qp_post_wr(qp);
- }
+ else if (post_sq)
+ irdma_uk_qp_post_wr(qp);
return 0;
}
@@ -805,62 +832,65 @@
struct irdma_post_sq_info *info, bool post_sq)
{
__le64 *wqe;
- struct irdma_post_inline_send *op_info;
+ struct irdma_post_send *op_info;
u64 hdr;
u32 wqe_idx;
bool read_fence = false;
u16 quanta;
+ u32 i, total_size = 0;
info->push_wqe = qp->push_db ? true : false;
- op_info = &info->op.inline_send;
+ op_info = &info->op.send;
- if (op_info->len > qp->max_inline_data)
+ if (unlikely(qp->max_sq_frag_cnt < op_info->num_sges))
return EINVAL;
- quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(op_info->len);
- wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, op_info->len,
- info);
+ for (i = 0; i < op_info->num_sges; i++)
+ total_size += op_info->sg_list[i].len;
+
+ if (unlikely(total_size > qp->max_inline_data))
+ return EINVAL;
+
+ quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(total_size);
+ wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, info);
if (!wqe)
return ENOSPC;
- irdma_clr_wqes(qp, wqe_idx);
-
set_64bit_val(wqe, IRDMA_BYTE_16,
- LS_64(op_info->qkey, IRDMAQPSQ_DESTQKEY) |
- LS_64(op_info->dest_qp, IRDMAQPSQ_DESTQPN));
+ FIELD_PREP(IRDMAQPSQ_DESTQKEY, op_info->qkey) |
+ FIELD_PREP(IRDMAQPSQ_DESTQPN, op_info->dest_qp));
read_fence |= info->read_fence;
- hdr = LS_64(info->stag_to_inv, IRDMAQPSQ_REMSTAG) |
- LS_64(op_info->ah_id, IRDMAQPSQ_AHID) |
- LS_64(info->op_type, IRDMAQPSQ_OPCODE) |
- LS_64(op_info->len, IRDMAQPSQ_INLINEDATALEN) |
- LS_64((info->imm_data_valid ? 1 : 0), IRDMAQPSQ_IMMDATAFLAG) |
- LS_64((info->report_rtt ? 1 : 0), IRDMAQPSQ_REPORTRTT) |
- LS_64(1, IRDMAQPSQ_INLINEDATAFLAG) |
- LS_64((info->push_wqe ? 1 : 0), IRDMAQPSQ_PUSHWQE) |
- LS_64(read_fence, IRDMAQPSQ_READFENCE) |
- LS_64(info->local_fence, IRDMAQPSQ_LOCALFENCE) |
- LS_64(info->signaled, IRDMAQPSQ_SIGCOMPL) |
- LS_64(info->udp_hdr, IRDMAQPSQ_UDPHEADER) |
- LS_64(info->l4len, IRDMAQPSQ_L4LEN) |
- LS_64(qp->swqe_polarity, IRDMAQPSQ_VALID);
+ hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, info->stag_to_inv) |
+ FIELD_PREP(IRDMAQPSQ_AHID, op_info->ah_id) |
+ FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) |
+ FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, total_size) |
+ FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG,
+ (info->imm_data_valid ? 1 : 0)) |
+ FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) |
+ FIELD_PREP(IRDMAQPSQ_INLINEDATAFLAG, 1) |
+ FIELD_PREP(IRDMAQPSQ_PUSHWQE, info->push_wqe) |
+ FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) |
+ FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) |
+ FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
+ FIELD_PREP(IRDMAQPSQ_UDPHEADER, info->udp_hdr) |
+ FIELD_PREP(IRDMAQPSQ_L4LEN, info->l4len) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
if (info->imm_data_valid)
set_64bit_val(wqe, IRDMA_BYTE_0,
- LS_64(info->imm_data, IRDMAQPSQ_IMMDATA));
- qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->data, op_info->len,
- qp->swqe_polarity);
+ FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data));
+ qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->sg_list,
+ op_info->num_sges, qp->swqe_polarity);
udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
- if (info->push_wqe) {
+ if (info->push_wqe)
irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);
- } else {
- if (post_sq)
- irdma_uk_qp_post_wr(qp);
- }
+ else if (post_sq)
+ irdma_uk_qp_post_wr(qp);
return 0;
}
@@ -882,41 +912,36 @@
u32 wqe_idx;
bool local_fence = false;
struct irdma_sge sge = {0};
+ u16 quanta = IRDMA_QP_WQE_MIN_QUANTA;
info->push_wqe = qp->push_db ? true : false;
op_info = &info->op.inv_local_stag;
local_fence = info->local_fence;
- wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, IRDMA_QP_WQE_MIN_QUANTA,
- 0, info);
+ wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, 0, info);
if (!wqe)
return ENOSPC;
- irdma_clr_wqes(qp, wqe_idx);
-
sge.stag = op_info->target_stag;
qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0, &sge, 0);
set_64bit_val(wqe, IRDMA_BYTE_16, 0);
- hdr = LS_64(IRDMA_OP_TYPE_INV_STAG, IRDMAQPSQ_OPCODE) |
- LS_64((info->push_wqe ? 1 : 0), IRDMAQPSQ_PUSHWQE) |
- LS_64(info->read_fence, IRDMAQPSQ_READFENCE) |
- LS_64(local_fence, IRDMAQPSQ_LOCALFENCE) |
- LS_64(info->signaled, IRDMAQPSQ_SIGCOMPL) |
- LS_64(qp->swqe_polarity, IRDMAQPSQ_VALID);
+ hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMA_OP_TYPE_INV_STAG) |
+ FIELD_PREP(IRDMAQPSQ_PUSHWQE, info->push_wqe) |
+ FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) |
+ FIELD_PREP(IRDMAQPSQ_LOCALFENCE, local_fence) |
+ FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
- if (info->push_wqe) {
- irdma_qp_push_wqe(qp, wqe, IRDMA_QP_WQE_MIN_QUANTA, wqe_idx,
- post_sq);
- } else {
- if (post_sq)
- irdma_uk_qp_post_wr(qp);
- }
+ if (info->push_wqe)
+ irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);
+ else if (post_sq)
+ irdma_uk_qp_post_wr(qp);
return 0;
}
@@ -936,44 +961,39 @@
u64 hdr;
u32 wqe_idx;
bool local_fence;
+ u16 quanta = IRDMA_QP_WQE_MIN_QUANTA;
info->push_wqe = qp->push_db ? true : false;
op_info = &info->op.bind_window;
local_fence = info->local_fence;
- wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, IRDMA_QP_WQE_MIN_QUANTA,
- 0, info);
+ wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, 0, info);
if (!wqe)
return ENOSPC;
- irdma_clr_wqes(qp, wqe_idx);
-
qp->wqe_ops.iw_set_mw_bind_wqe(wqe, op_info);
- hdr = LS_64(IRDMA_OP_TYPE_BIND_MW, IRDMAQPSQ_OPCODE) |
- LS_64(((op_info->ena_reads << 2) | (op_info->ena_writes << 3)),
- IRDMAQPSQ_STAGRIGHTS) |
- LS_64((op_info->addressing_type == IRDMA_ADDR_TYPE_VA_BASED ? 1 : 0),
- IRDMAQPSQ_VABASEDTO) |
- LS_64((op_info->mem_window_type_1 ? 1 : 0),
- IRDMAQPSQ_MEMWINDOWTYPE) |
- LS_64((info->push_wqe ? 1 : 0), IRDMAQPSQ_PUSHWQE) |
- LS_64(info->read_fence, IRDMAQPSQ_READFENCE) |
- LS_64(local_fence, IRDMAQPSQ_LOCALFENCE) |
- LS_64(info->signaled, IRDMAQPSQ_SIGCOMPL) |
- LS_64(qp->swqe_polarity, IRDMAQPSQ_VALID);
+ hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMA_OP_TYPE_BIND_MW) |
+ FIELD_PREP(IRDMAQPSQ_STAGRIGHTS,
+ ((op_info->ena_reads << 2) | (op_info->ena_writes << 3))) |
+ FIELD_PREP(IRDMAQPSQ_VABASEDTO,
+ (op_info->addressing_type == IRDMA_ADDR_TYPE_VA_BASED ? 1 : 0)) |
+ FIELD_PREP(IRDMAQPSQ_MEMWINDOWTYPE,
+ (op_info->mem_window_type_1 ? 1 : 0)) |
+ FIELD_PREP(IRDMAQPSQ_PUSHWQE, info->push_wqe) |
+ FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) |
+ FIELD_PREP(IRDMAQPSQ_LOCALFENCE, local_fence) |
+ FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
- if (info->push_wqe) {
- irdma_qp_push_wqe(qp, wqe, IRDMA_QP_WQE_MIN_QUANTA, wqe_idx,
- post_sq);
- } else {
- if (post_sq)
- irdma_uk_qp_post_wr(qp);
- }
+ if (info->push_wqe)
+ irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);
+ else if (post_sq)
+ irdma_uk_qp_post_wr(qp);
return 0;
}
@@ -1020,8 +1040,8 @@
}
set_64bit_val(wqe, IRDMA_BYTE_16, 0);
- hdr = LS_64(addl_frag_cnt, IRDMAQPSQ_ADDFRAGCNT) |
- LS_64(qp->rwqe_polarity, IRDMAQPSQ_VALID);
+ hdr = FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->rwqe_polarity);
udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
@@ -1061,17 +1081,17 @@
get_64bit_val(cq->shadow_area, 32, &temp_val);
- sw_cq_sel = (u16)RS_64(temp_val, IRDMA_CQ_DBSA_SW_CQ_SELECT);
+ sw_cq_sel = (u16)FIELD_GET(IRDMA_CQ_DBSA_SW_CQ_SELECT, temp_val);
sw_cq_sel += cq_cnt;
- arm_seq_num = (u8)RS_64(temp_val, IRDMA_CQ_DBSA_ARM_SEQ_NUM);
- arm_next_se = (u8)RS_64(temp_val, IRDMA_CQ_DBSA_ARM_NEXT_SE);
- arm_next = (u8)RS_64(temp_val, IRDMA_CQ_DBSA_ARM_NEXT);
+ arm_seq_num = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_SEQ_NUM, temp_val);
+ arm_next_se = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT_SE, temp_val);
+ arm_next = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT, temp_val);
- temp_val = LS_64(arm_seq_num, IRDMA_CQ_DBSA_ARM_SEQ_NUM) |
- LS_64(sw_cq_sel, IRDMA_CQ_DBSA_SW_CQ_SELECT) |
- LS_64(arm_next_se, IRDMA_CQ_DBSA_ARM_NEXT_SE) |
- LS_64(arm_next, IRDMA_CQ_DBSA_ARM_NEXT);
+ temp_val = FIELD_PREP(IRDMA_CQ_DBSA_ARM_SEQ_NUM, arm_seq_num) |
+ FIELD_PREP(IRDMA_CQ_DBSA_SW_CQ_SELECT, sw_cq_sel) |
+ FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT_SE, arm_next_se) |
+ FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT, arm_next);
set_64bit_val(cq->shadow_area, 32, temp_val);
}
@@ -1093,17 +1113,17 @@
cq->armed = true;
get_64bit_val(cq->shadow_area, IRDMA_BYTE_32, &temp_val);
- arm_seq_num = (u8)RS_64(temp_val, IRDMA_CQ_DBSA_ARM_SEQ_NUM);
+ arm_seq_num = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_SEQ_NUM, temp_val);
arm_seq_num++;
- sw_cq_sel = (u16)RS_64(temp_val, IRDMA_CQ_DBSA_SW_CQ_SELECT);
- arm_next_se = (u8)RS_64(temp_val, IRDMA_CQ_DBSA_ARM_NEXT_SE);
+ sw_cq_sel = (u16)FIELD_GET(IRDMA_CQ_DBSA_SW_CQ_SELECT, temp_val);
+ arm_next_se = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT_SE, temp_val);
arm_next_se |= 1;
if (cq_notify == IRDMA_CQ_COMPL_EVENT)
arm_next = 1;
- temp_val = LS_64(arm_seq_num, IRDMA_CQ_DBSA_ARM_SEQ_NUM) |
- LS_64(sw_cq_sel, IRDMA_CQ_DBSA_SW_CQ_SELECT) |
- LS_64(arm_next_se, IRDMA_CQ_DBSA_ARM_NEXT_SE) |
- LS_64(arm_next, IRDMA_CQ_DBSA_ARM_NEXT);
+ temp_val = FIELD_PREP(IRDMA_CQ_DBSA_ARM_SEQ_NUM, arm_seq_num) |
+ FIELD_PREP(IRDMA_CQ_DBSA_SW_CQ_SELECT, sw_cq_sel) |
+ FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT_SE, arm_next_se) |
+ FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT, arm_next);
set_64bit_val(cq->shadow_area, IRDMA_BYTE_32, temp_val);
@@ -1123,12 +1143,12 @@
get_64bit_val(src, offset + 8, &val);
if (flip)
- val ^= IRDMAQPSQ_VALID_M;
+ val ^= IRDMAQPSQ_VALID;
set_64bit_val(dst, offset + 8, val);
get_64bit_val(src, offset + 24, &val);
if (flip)
- val ^= IRDMAQPSQ_VALID_M;
+ val ^= IRDMAQPSQ_VALID;
if (barrier)
udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
set_64bit_val(dst, offset + 24, val);
@@ -1160,7 +1180,8 @@
u64 val;
libirdma_debug("reposting_wqes: from start_idx=%d to end_idx = %d\n", start_idx, end_idx);
- pthread_spin_lock(qp->lock);
+ if (pthread_spin_lock(qp->lock))
+ return;
while (start_idx != end_idx) {
IRDMA_RING_SET_TAIL(qp->rq_ring, start_idx + 1);
src_wqe = qp->rq_base[start_idx * qp->rq_wqe_size_multiplier].elem;
@@ -1168,7 +1189,7 @@
/* Check to see if polarity has changed */
get_64bit_val(src_wqe, 24, &val);
- if (RS_64(val, IRDMAQPSQ_VALID) != qp->rwqe_polarity)
+ if (FIELD_GET(IRDMAQPSQ_VALID, val) != qp->rwqe_polarity)
flip_polarity = true;
else
flip_polarity = false;
@@ -1242,7 +1263,7 @@
__le64 *cqe;
struct irdma_qp_uk *qp;
struct irdma_ring *pring = NULL;
- u32 wqe_idx, q_type;
+ u32 wqe_idx;
int ret_code;
bool move_cq_head = true;
u8 polarity;
@@ -1255,14 +1276,14 @@
cqe = IRDMA_GET_CURRENT_CQ_ELEM(cq);
get_64bit_val(cqe, IRDMA_BYTE_24, &qword3);
- polarity = (u8)RS_64(qword3, IRDMA_CQ_VALID);
+ polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3);
if (polarity != cq->polarity)
return ENOENT;
/* Ensure CQE contents are read after valid bit is checked */
udma_from_device_barrier();
- ext_valid = (bool)RS_64(qword3, IRDMA_CQ_EXTCQE);
+ ext_valid = (bool)FIELD_GET(IRDMA_CQ_EXTCQE, qword3);
if (ext_valid) {
u64 qword6, qword7;
u32 peek_head;
@@ -1270,12 +1291,12 @@
if (cq->avoid_mem_cflct) {
ext_cqe = (__le64 *) ((u8 *)cqe + 32);
get_64bit_val(ext_cqe, IRDMA_BYTE_24, &qword7);
- polarity = (u8)RS_64(qword7, IRDMA_CQ_VALID);
+ polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword7);
} else {
peek_head = (cq->cq_ring.head + 1) % cq->cq_ring.size;
ext_cqe = cq->cq_base[peek_head].buf;
get_64bit_val(ext_cqe, IRDMA_BYTE_24, &qword7);
- polarity = (u8)RS_64(qword7, IRDMA_CQ_VALID);
+ polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword7);
if (!peek_head)
polarity ^= 1;
}
@@ -1285,19 +1306,19 @@
/* Ensure ext CQE contents are read after ext valid bit is checked */
udma_from_device_barrier();
- info->imm_valid = (bool)RS_64(qword7, IRDMA_CQ_IMMVALID);
+ info->imm_valid = (bool)FIELD_GET(IRDMA_CQ_IMMVALID, qword7);
if (info->imm_valid) {
u64 qword4;
get_64bit_val(ext_cqe, IRDMA_BYTE_0, &qword4);
- info->imm_data = (u32)RS_64(qword4, IRDMA_CQ_IMMDATALOW32);
+ info->imm_data = (u32)FIELD_GET(IRDMA_CQ_IMMDATALOW32, qword4);
}
- info->ud_smac_valid = (bool)RS_64(qword7, IRDMA_CQ_UDSMACVALID);
- info->ud_vlan_valid = (bool)RS_64(qword7, IRDMA_CQ_UDVLANVALID);
+ info->ud_smac_valid = (bool)FIELD_GET(IRDMA_CQ_UDSMACVALID, qword7);
+ info->ud_vlan_valid = (bool)FIELD_GET(IRDMA_CQ_UDVLANVALID, qword7);
if (info->ud_smac_valid || info->ud_vlan_valid) {
get_64bit_val(ext_cqe, IRDMA_BYTE_16, &qword6);
if (info->ud_vlan_valid)
- info->ud_vlan = (u16)RS_64(qword6, IRDMA_CQ_UDVLAN);
+ info->ud_vlan = (u16)FIELD_GET(IRDMA_CQ_UDVLAN, qword6);
if (info->ud_smac_valid) {
info->ud_smac[5] = qword6 & 0xFF;
info->ud_smac[4] = (qword6 >> 8) & 0xFF;
@@ -1313,23 +1334,26 @@
info->ud_vlan_valid = false;
}
- q_type = (u8)RS_64(qword3, IRDMA_CQ_SQ);
- info->error = (bool)RS_64(qword3, IRDMA_CQ_ERROR);
- info->push_dropped = (bool)RS_64(qword3, IRDMACQ_PSHDROP);
- info->ipv4 = (bool)RS_64(qword3, IRDMACQ_IPV4);
+ info->q_type = (u8)FIELD_GET(IRDMA_CQ_SQ, qword3);
+ info->error = (bool)FIELD_GET(IRDMA_CQ_ERROR, qword3);
+ info->push_dropped = (bool)FIELD_GET(IRDMACQ_PSHDROP, qword3);
+ info->ipv4 = (bool)FIELD_GET(IRDMACQ_IPV4, qword3);
if (info->error) {
- info->major_err = RS_64(qword3, IRDMA_CQ_MAJERR);
- info->minor_err = RS_64(qword3, IRDMA_CQ_MINERR);
- if (info->major_err == IRDMA_FLUSH_MAJOR_ERR) {
- info->comp_status = IRDMA_COMPL_STATUS_FLUSHED;
+ info->major_err = FIELD_GET(IRDMA_CQ_MAJERR, qword3);
+ info->minor_err = FIELD_GET(IRDMA_CQ_MINERR, qword3);
+ switch (info->major_err) {
+ case IRDMA_FLUSH_MAJOR_ERR:
/* Set the min error to standard flush error code for remaining cqes */
if (info->minor_err != FLUSH_GENERAL_ERR) {
- qword3 &= ~IRDMA_CQ_MINERR_M;
- qword3 |= LS_64(FLUSH_GENERAL_ERR, IRDMA_CQ_MINERR);
+ qword3 &= ~IRDMA_CQ_MINERR;
+ qword3 |= FIELD_PREP(IRDMA_CQ_MINERR, FLUSH_GENERAL_ERR);
set_64bit_val(cqe, IRDMA_BYTE_24, qword3);
}
- } else {
+ info->comp_status = IRDMA_COMPL_STATUS_FLUSHED;
+ break;
+ default:
info->comp_status = IRDMA_COMPL_STATUS_UNKNOWN;
+ break;
}
} else {
info->comp_status = IRDMA_COMPL_STATUS_SUCCESS;
@@ -1338,22 +1362,23 @@
get_64bit_val(cqe, IRDMA_BYTE_0, &qword0);
get_64bit_val(cqe, IRDMA_BYTE_16, &qword2);
- info->tcp_seq_num_rtt = (u32)RS_64(qword0, IRDMACQ_TCPSEQNUMRTT);
- info->qp_id = (u32)RS_64(qword2, IRDMACQ_QPID);
- info->ud_src_qpn = (u32)RS_64(qword2, IRDMACQ_UDSRCQPN);
+ info->stat.raw = (u32)FIELD_GET(IRDMACQ_TCPSQN_ROCEPSN_RTT_TS, qword0);
+ info->qp_id = (u32)FIELD_GET(IRDMACQ_QPID, qword2);
+ info->ud_src_qpn = (u32)FIELD_GET(IRDMACQ_UDSRCQPN, qword2);
get_64bit_val(cqe, IRDMA_BYTE_8, &comp_ctx);
- info->solicited_event = (bool)RS_64(qword3, IRDMACQ_SOEVENT);
+ info->solicited_event = (bool)FIELD_GET(IRDMACQ_SOEVENT, qword3);
qp = (struct irdma_qp_uk *)(irdma_uintptr) comp_ctx;
if (!qp || qp->destroy_pending) {
ret_code = EFAULT;
goto exit;
}
- wqe_idx = (u32)RS_64(qword3, IRDMA_CQ_WQEIDX);
+ wqe_idx = (u32)FIELD_GET(IRDMA_CQ_WQEIDX, qword3);
info->qp_handle = (irdma_qp_handle) (irdma_uintptr) qp;
+ info->op_type = (u8)FIELD_GET(IRDMACQ_OP, qword3);
- if (q_type == IRDMA_CQE_QTYPE_RQ) {
+ if (info->q_type == IRDMA_CQE_QTYPE_RQ) {
u32 array_idx;
ret_code = irdma_skip_duplicate_flush_cmpl(qp->rq_ring,
@@ -1386,16 +1411,11 @@
}
}
- info->bytes_xfered = (u32)RS_64(qword0, IRDMACQ_PAYLDLEN);
+ info->bytes_xfered = (u32)FIELD_GET(IRDMACQ_PAYLDLEN, qword0);
- if (info->imm_valid)
- info->op_type = IRDMA_OP_TYPE_REC_IMM;
- else
- info->op_type = IRDMA_OP_TYPE_REC;
-
- if (qword3 & IRDMACQ_STAG_M) {
+ if (qword3 & IRDMACQ_STAG) {
info->stag_invalid_set = true;
- info->inv_stag = (u32)RS_64(qword2, IRDMACQ_INVSTAG);
+ info->inv_stag = (u32)FIELD_GET(IRDMACQ_INVSTAG, qword2);
} else {
info->stag_invalid_set = false;
}
@@ -1439,11 +1459,16 @@
info->signaled = qp->sq_wrtrk_array[wqe_idx].signaled;
if (!info->comp_status)
info->bytes_xfered = qp->sq_wrtrk_array[wqe_idx].wr_len;
- info->op_type = (u8)RS_64(qword3, IRDMACQ_OP);
+ info->op_type = (u8)FIELD_GET(IRDMACQ_OP, qword3);
IRDMA_RING_SET_TAIL(qp->sq_ring,
wqe_idx + qp->sq_wrtrk_array[wqe_idx].quanta);
} else {
+ if (pthread_spin_lock(qp->lock)) {
+ ret_code = ENOENT;
+ goto exit;
+ }
if (!IRDMA_RING_MORE_WORK(qp->sq_ring)) {
+ pthread_spin_unlock(qp->lock);
ret_code = ENOENT;
goto exit;
}
@@ -1451,27 +1476,29 @@
do {
__le64 *sw_wqe;
u64 wqe_qword;
- u8 op_type;
u32 tail;
tail = qp->sq_ring.tail;
sw_wqe = qp->sq_base[tail].elem;
get_64bit_val(sw_wqe, IRDMA_BYTE_24,
&wqe_qword);
- op_type = (u8)RS_64(wqe_qword, IRDMAQPSQ_OPCODE);
- info->op_type = op_type;
+ info->op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, wqe_qword);
IRDMA_RING_SET_TAIL(qp->sq_ring,
tail + qp->sq_wrtrk_array[tail].quanta);
- if (op_type != IRDMAQP_OP_NOP) {
+ if (info->op_type != IRDMAQP_OP_NOP) {
info->wr_id = qp->sq_wrtrk_array[tail].wrid;
info->signaled = qp->sq_wrtrk_array[tail].signaled;
info->bytes_xfered = qp->sq_wrtrk_array[tail].wr_len;
break;
}
} while (1);
+
+ if (info->op_type == IRDMA_OP_TYPE_BIND_MW && info->minor_err == FLUSH_PROT_ERR)
+ info->minor_err = FLUSH_MW_BIND_ERR;
qp->sq_flush_seen = true;
if (!IRDMA_RING_MORE_WORK(qp->sq_ring))
qp->sq_flush_complete = true;
+ pthread_spin_unlock(qp->lock);
}
pring = &qp->sq_ring;
}
@@ -1479,9 +1506,10 @@
ret_code = 0;
exit:
- if (!ret_code && info->comp_status == IRDMA_COMPL_STATUS_FLUSHED)
+ if (!ret_code && info->comp_status == IRDMA_COMPL_STATUS_FLUSHED) {
if (pring && IRDMA_RING_MORE_WORK(*pring))
move_cq_head = false;
+ }
if (move_cq_head) {
IRDMA_RING_MOVE_HEAD_NOCHECK(cq->cq_ring);
@@ -1500,8 +1528,8 @@
set_64bit_val(cq->shadow_area, IRDMA_BYTE_0,
IRDMA_RING_CURRENT_HEAD(cq->cq_ring));
} else {
- qword3 &= ~IRDMA_CQ_WQEIDX_M;
- qword3 |= LS_64(pring->tail, IRDMA_CQ_WQEIDX);
+ qword3 &= ~IRDMA_CQ_WQEIDX;
+ qword3 |= FIELD_PREP(IRDMA_CQ_WQEIDX, pring->tail);
set_64bit_val(cqe, IRDMA_BYTE_24, qword3);
}
@@ -1509,11 +1537,11 @@
}
/**
- * irdma_qp_round_up - return round up qp wq depth
+ * irdma_round_up_wq - return round up qp wq depth
* @wqdepth: wq depth in quanta to round up
*/
static int
-irdma_qp_round_up(u32 wqdepth)
+irdma_round_up_wq(u32 wqdepth)
{
int scount = 1;
@@ -1557,34 +1585,34 @@
}
/*
- * irdma_get_sqdepth - get SQ depth (quanta) @max_hw_wq_quanta: HW SQ size limit @sq_size: SQ size @shift: shift which
+ * irdma_get_sqdepth - get SQ depth (quanta) @uk_attrs: qp HW attributes @sq_size: SQ size @shift: shift which
* determines size of WQE @sqdepth: depth of SQ
*/
int
-irdma_get_sqdepth(u32 max_hw_wq_quanta, u32 sq_size, u8 shift, u32 *sqdepth)
+irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift, u32 *sqdepth)
{
- *sqdepth = irdma_qp_round_up((sq_size << shift) + IRDMA_SQ_RSVD);
+ *sqdepth = irdma_round_up_wq((sq_size << shift) + IRDMA_SQ_RSVD);
- if (*sqdepth < (IRDMA_QP_SW_MIN_WQSIZE << shift))
- *sqdepth = IRDMA_QP_SW_MIN_WQSIZE << shift;
- else if (*sqdepth > max_hw_wq_quanta)
+ if (*sqdepth < ((u32)uk_attrs->min_hw_wq_size << shift))
+ *sqdepth = uk_attrs->min_hw_wq_size << shift;
+ else if (*sqdepth > uk_attrs->max_hw_wq_quanta)
return EINVAL;
return 0;
}
/*
- * irdma_get_rqdepth - get RQ/SRQ depth (quanta) @max_hw_rq_quanta: HW RQ/SRQ size limit @rq_size: RQ/SRQ size @shift:
- * shift which determines size of WQE @rqdepth: depth of RQ/SRQ
+ * irdma_get_rqdepth - get RQ depth (quanta) @uk_attrs: qp HW attributes @rq_size: SRQ size @shift: shift which
+ * determines size of WQE @rqdepth: depth of RQ/SRQ
*/
int
-irdma_get_rqdepth(u32 max_hw_rq_quanta, u32 rq_size, u8 shift, u32 *rqdepth)
+irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift, u32 *rqdepth)
{
- *rqdepth = irdma_qp_round_up((rq_size << shift) + IRDMA_RQ_RSVD);
+ *rqdepth = irdma_round_up_wq((rq_size << shift) + IRDMA_RQ_RSVD);
- if (*rqdepth < (IRDMA_QP_SW_MIN_WQSIZE << shift))
- *rqdepth = IRDMA_QP_SW_MIN_WQSIZE << shift;
- else if (*rqdepth > max_hw_rq_quanta)
+ if (*rqdepth < ((u32)uk_attrs->min_hw_wq_size << shift))
+ *rqdepth = uk_attrs->min_hw_wq_size << shift;
+ else if (*rqdepth > uk_attrs->max_hw_rq_quanta)
return EINVAL;
return 0;
@@ -1625,6 +1653,80 @@
IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->initial_ring, move_cnt);
}
+/**
+ * irdma_uk_calc_shift_wq - calculate WQE shift for both SQ and RQ
+ * @ukinfo: qp initialization info
+ * @sq_shift: Returns shift of SQ
+ * @rq_shift: Returns shift of RQ
+ */
+void
+irdma_uk_calc_shift_wq(struct irdma_qp_uk_init_info *ukinfo, u8 *sq_shift,
+ u8 *rq_shift)
+{
+ bool imm_support = ukinfo->uk_attrs->hw_rev >= IRDMA_GEN_2 ? true : false;
+
+ irdma_get_wqe_shift(ukinfo->uk_attrs,
+ imm_support ? ukinfo->max_sq_frag_cnt + 1 :
+ ukinfo->max_sq_frag_cnt,
+ ukinfo->max_inline_data, sq_shift);
+
+ irdma_get_wqe_shift(ukinfo->uk_attrs, ukinfo->max_rq_frag_cnt, 0,
+ rq_shift);
+
+ if (ukinfo->uk_attrs->hw_rev == IRDMA_GEN_1) {
+ if (ukinfo->abi_ver > 4)
+ *rq_shift = IRDMA_MAX_RQ_WQE_SHIFT_GEN1;
+ }
+}
+
+/**
+ * irdma_uk_calc_depth_shift_sq - calculate depth and shift for SQ size.
+ * @ukinfo: qp initialization info
+ * @sq_depth: Returns depth of SQ
+ * @sq_shift: Returns shift of SQ
+ */
+int
+irdma_uk_calc_depth_shift_sq(struct irdma_qp_uk_init_info *ukinfo,
+ u32 *sq_depth, u8 *sq_shift)
+{
+ bool imm_support = ukinfo->uk_attrs->hw_rev >= IRDMA_GEN_2 ? true : false;
+ int status;
+ irdma_get_wqe_shift(ukinfo->uk_attrs,
+ imm_support ? ukinfo->max_sq_frag_cnt + 1 :
+ ukinfo->max_sq_frag_cnt,
+ ukinfo->max_inline_data, sq_shift);
+ status = irdma_get_sqdepth(ukinfo->uk_attrs, ukinfo->sq_size,
+ *sq_shift, sq_depth);
+
+ return status;
+}
+
+/**
+ * irdma_uk_calc_depth_shift_rq - calculate depth and shift for RQ size.
+ * @ukinfo: qp initialization info
+ * @rq_depth: Returns depth of RQ
+ * @rq_shift: Returns shift of RQ
+ */
+int
+irdma_uk_calc_depth_shift_rq(struct irdma_qp_uk_init_info *ukinfo,
+ u32 *rq_depth, u8 *rq_shift)
+{
+ int status;
+
+ irdma_get_wqe_shift(ukinfo->uk_attrs, ukinfo->max_rq_frag_cnt, 0,
+ rq_shift);
+
+ if (ukinfo->uk_attrs->hw_rev == IRDMA_GEN_1) {
+ if (ukinfo->abi_ver > 4)
+ *rq_shift = IRDMA_MAX_RQ_WQE_SHIFT_GEN1;
+ }
+
+ status = irdma_get_rqdepth(ukinfo->uk_attrs, ukinfo->rq_size,
+ *rq_shift, rq_depth);
+
+ return status;
+}
+
/**
* irdma_uk_qp_init - initialize shared qp
* @qp: hw qp (user and kernel)
@@ -1640,23 +1742,12 @@
{
int ret_code = 0;
u32 sq_ring_size;
- u8 sqshift, rqshift;
qp->uk_attrs = info->uk_attrs;
if (info->max_sq_frag_cnt > qp->uk_attrs->max_hw_wq_frags ||
info->max_rq_frag_cnt > qp->uk_attrs->max_hw_wq_frags)
return EINVAL;
- irdma_get_wqe_shift(qp->uk_attrs, info->max_rq_frag_cnt, 0, &rqshift);
- if (qp->uk_attrs->hw_rev == IRDMA_GEN_1) {
- irdma_get_wqe_shift(qp->uk_attrs, info->max_sq_frag_cnt,
- info->max_inline_data, &sqshift);
- if (info->abi_ver > 4)
- rqshift = IRDMA_MAX_RQ_WQE_SHIFT_GEN1;
- } else {
- irdma_get_wqe_shift(qp->uk_attrs, info->max_sq_frag_cnt + 1,
- info->max_inline_data, &sqshift);
- }
qp->qp_caps = info->qp_caps;
qp->sq_base = info->sq;
qp->rq_base = info->rq;
@@ -1672,7 +1763,7 @@
qp->sq_size = info->sq_size;
qp->push_mode = false;
qp->max_sq_frag_cnt = info->max_sq_frag_cnt;
- sq_ring_size = qp->sq_size << sqshift;
+ sq_ring_size = qp->sq_size << info->sq_shift;
IRDMA_RING_INIT(qp->sq_ring, sq_ring_size);
IRDMA_RING_INIT(qp->initial_ring, sq_ring_size);
if (info->first_sq_wq) {
@@ -1687,9 +1778,9 @@
qp->rq_size = info->rq_size;
qp->max_rq_frag_cnt = info->max_rq_frag_cnt;
qp->max_inline_data = info->max_inline_data;
- qp->rq_wqe_size = rqshift;
+ qp->rq_wqe_size = info->rq_shift;
IRDMA_RING_INIT(qp->rq_ring, qp->rq_size);
- qp->rq_wqe_size_multiplier = 1 << rqshift;
+ qp->rq_wqe_size_multiplier = 1 << info->rq_shift;
if (qp->uk_attrs->hw_rev == IRDMA_GEN_1)
qp->wqe_ops = iw_wqe_uk_ops_gen_1;
else
@@ -1739,7 +1830,7 @@
else
cqe = cq->cq_base[cq_head].buf;
get_64bit_val(cqe, IRDMA_BYTE_24, &qword3);
- polarity = (u8)RS_64(qword3, IRDMA_CQ_VALID);
+ polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3);
if (polarity != temp)
break;
@@ -1769,28 +1860,29 @@
u64 hdr;
u32 wqe_idx;
struct irdma_post_sq_info info = {0};
+ u16 quanta = IRDMA_QP_WQE_MIN_QUANTA;
- info.push_wqe = false;
+ info.push_wqe = qp->push_db ? true : false;
info.wr_id = wr_id;
- wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, IRDMA_QP_WQE_MIN_QUANTA,
- 0, &info);
+ wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, 0, &info);
if (!wqe)
return ENOSPC;
- irdma_clr_wqes(qp, wqe_idx);
-
set_64bit_val(wqe, IRDMA_BYTE_0, 0);
set_64bit_val(wqe, IRDMA_BYTE_8, 0);
set_64bit_val(wqe, IRDMA_BYTE_16, 0);
- hdr = LS_64(IRDMAQP_OP_NOP, IRDMAQPSQ_OPCODE) |
- LS_64(signaled, IRDMAQPSQ_SIGCOMPL) |
- LS_64(qp->swqe_polarity, IRDMAQPSQ_VALID);
+ hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) |
+ FIELD_PREP(IRDMAQPSQ_SIGCOMPL, signaled) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
- if (post_sq)
+
+ if (info.push_wqe)
+ irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);
+ else if (post_sq)
irdma_uk_qp_post_wr(qp);
return 0;
diff --git a/contrib/ofed/libirdma/irdma_umain.h b/contrib/ofed/libirdma/irdma_umain.h
--- a/contrib/ofed/libirdma/irdma_umain.h
+++ b/contrib/ofed/libirdma/irdma_umain.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (C) 2019 - 2020 Intel Corporation
+ * Copyright (C) 2019 - 2022 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -46,12 +46,6 @@
#include "i40iw_hw.h"
#include "irdma_user.h"
-#ifndef likely
-#define likely(x) __builtin_expect((x), 1)
-#endif
-#ifndef unlikely
-#define unlikely(x) __builtin_expect((x), 0)
-#endif
#define PFX "libirdma-"
#define IRDMA_BASE_PUSH_PAGE 1
@@ -62,13 +56,12 @@
LIST_HEAD(list_head, irdma_cq_buf);
LIST_HEAD(list_head_cmpl, irdma_cmpl_gen);
-enum irdma_supported_wc_flags {
- IRDMA_CQ_SUPPORTED_WC_FLAGS = IBV_WC_EX_WITH_BYTE_LEN
+enum irdma_supported_wc_flags_ex {
+ IRDMA_STANDARD_WC_FLAGS_EX = IBV_WC_EX_WITH_BYTE_LEN
| IBV_WC_EX_WITH_IMM
| IBV_WC_EX_WITH_QP_NUM
| IBV_WC_EX_WITH_SRC_QP
- | IBV_WC_EX_WITH_SL
- | IBV_WC_EX_WITH_COMPLETION_TIMESTAMP,
+ | IBV_WC_EX_WITH_SL,
};
struct irdma_udevice {
@@ -103,8 +96,11 @@
LIST_ENTRY(irdma_cq_buf) list;
struct irdma_cq_uk cq;
struct verbs_mr vmr;
+ size_t buf_size;
};
+extern pthread_mutex_t sigusr1_wait_mutex;
+
struct verbs_cq {
union {
struct ibv_cq cq;
@@ -128,7 +124,6 @@
bool arm_sol;
bool skip_sol;
int comp_vector;
- uint32_t report_rtt;
struct irdma_uqp *uqp;
struct irdma_cq_uk cq;
struct list_head resize_list;
@@ -139,7 +134,6 @@
struct irdma_uqp {
struct ibv_qp ibv_qp;
- struct ibv_qp_attr attr;
struct irdma_ucq *send_cq;
struct irdma_ucq *recv_cq;
struct verbs_mr vmr;
@@ -154,16 +148,10 @@
struct ibv_recv_wr *pend_rx_wr;
struct irdma_qp_uk qp;
enum ibv_qp_type qp_type;
- enum ibv_qp_attr_mask attr_mask;
struct irdma_sge *recv_sges;
pthread_t flush_thread;
};
-struct irdma_umr {
- struct verbs_mr vmr;
- uint32_t acc_flags;
-};
-
/* irdma_uverbs.c */
int irdma_uquery_device_ex(struct ibv_context *context,
const struct ibv_query_device_ex_input *input,
@@ -176,6 +164,10 @@
struct ibv_mr *irdma_ureg_mr(struct ibv_pd *pd, void *addr, size_t length,
int access);
int irdma_udereg_mr(struct ibv_mr *mr);
+
+int irdma_urereg_mr(struct verbs_mr *mr, int flags, struct ibv_pd *pd, void *addr,
+ size_t length, int access);
+
struct ibv_mw *irdma_ualloc_mw(struct ibv_pd *pd, enum ibv_mw_type type);
int irdma_ubind_mw(struct ibv_qp *qp, struct ibv_mw *mw,
struct ibv_mw_bind *mw_bind);
diff --git a/contrib/ofed/libirdma/irdma_umain.c b/contrib/ofed/libirdma/irdma_umain.c
--- a/contrib/ofed/libirdma/irdma_umain.c
+++ b/contrib/ofed/libirdma/irdma_umain.c
@@ -39,6 +39,7 @@
#include <stdlib.h>
#include "irdma_umain.h"
#include "irdma-abi.h"
+#include "irdma_uquery.h"
#include "ice_devids.h"
#include "i40e_devids.h"
@@ -48,7 +49,7 @@
/**
* Driver version
*/
-char libirdma_version[] = "0.0.51-k";
+char libirdma_version[] = "1.1.5-k";
unsigned int irdma_dbg;
@@ -118,6 +119,28 @@
.detach_mcast = irdma_udetach_mcast,
};
+/**
+ * libirdma_query_device - fill libirdma_device structure
+ * @ctx_in - ibv_context identifying device
+ * @out - libirdma_device structure to fill quered info
+ *
+ * ctx_in is not used at the moment
+ */
+int
+libirdma_query_device(struct ibv_context *ctx_in, struct libirdma_device *out)
+{
+ if (!out)
+ return EIO;
+ if (sizeof(out->lib_ver) < sizeof(libirdma_version))
+ return ERANGE;
+
+ out->query_ver = 1;
+ snprintf(out->lib_ver, min(sizeof(libirdma_version), sizeof(out->lib_ver)),
+ "%s", libirdma_version);
+
+ return 0;
+}
+
static int
irdma_init_context(struct verbs_device *vdev,
struct ibv_context *ctx, int cmd_fd)
@@ -147,6 +170,7 @@
iwvctx->uk_attrs.max_hw_sq_chunk = resp.max_hw_sq_chunk;
iwvctx->uk_attrs.max_hw_cq_size = resp.max_hw_cq_size;
iwvctx->uk_attrs.min_hw_cq_size = resp.min_hw_cq_size;
+ iwvctx->uk_attrs.min_hw_wq_size = IRDMA_MIN_WQ_SIZE_GEN2;
iwvctx->abi_ver = IRDMA_ABI_VER;
mmap_key = resp.db_mmap_key;
@@ -180,8 +204,6 @@
{
struct irdma_uvcontext *iwvctx;
- printf("%s %s CALL\n", __FILE__, __func__);
-
iwvctx = container_of(ibctx, struct irdma_uvcontext, ibv_ctx);
irdma_ufree_pd(&iwvctx->iwupd->ibv_pd);
munmap(iwvctx->db, IRDMA_HW_PAGE_SIZE);
diff --git a/contrib/ofed/libirdma/irdma_uquery.h b/contrib/ofed/libirdma/irdma_uquery.h
new file mode 100644
--- /dev/null
+++ b/contrib/ofed/libirdma/irdma_uquery.h
@@ -0,0 +1,50 @@
+/*-
+ * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
+ *
+ * Copyright (C) 2022 Intel Corporation
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenFabrics.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+/*$FreeBSD$*/
+
+
+#ifndef IRDMA_UQUERY_H
+#define IRDMA_UQUERY_H
+
+#include <infiniband/verbs.h>
+#include "osdep.h"
+
+struct libirdma_device {
+ uint32_t query_ver;
+ char lib_ver[32];
+ uint8_t rsvd[128];
+};
+
+int libirdma_query_device(struct ibv_context *ctx_in, struct libirdma_device *out);
+#endif /* IRDMA_UQUERY_H */
diff --git a/contrib/ofed/libirdma/irdma_user.h b/contrib/ofed/libirdma/irdma_user.h
--- a/contrib/ofed/libirdma/irdma_user.h
+++ b/contrib/ofed/libirdma/irdma_user.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2021 Intel Corporation
+ * Copyright (c) 2015 - 2022 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -53,7 +53,7 @@
#define irdma_address_list u64 *
#define irdma_sgl struct irdma_sge *
-#define IRDMA_MAX_MR_SIZE 0x200000000000ULL
+#define IRDMA_MAX_MR_SIZE 0x200000000000ULL
#define IRDMA_ACCESS_FLAGS_LOCALREAD 0x01
#define IRDMA_ACCESS_FLAGS_LOCALWRITE 0x02
@@ -80,7 +80,97 @@
#define IRDMA_OP_TYPE_REC 0x3e
#define IRDMA_OP_TYPE_REC_IMM 0x3f
-#define IRDMA_FLUSH_MAJOR_ERR 1
+#define IRDMA_FLUSH_MAJOR_ERR 1
+#define IRDMA_SRQFLUSH_RSVD_MAJOR_ERR 0xfffe
+
+/* Async Events codes */
+#define IRDMA_AE_AMP_UNALLOCATED_STAG 0x0102
+#define IRDMA_AE_AMP_INVALID_STAG 0x0103
+#define IRDMA_AE_AMP_BAD_QP 0x0104
+#define IRDMA_AE_AMP_BAD_PD 0x0105
+#define IRDMA_AE_AMP_BAD_STAG_KEY 0x0106
+#define IRDMA_AE_AMP_BAD_STAG_INDEX 0x0107
+#define IRDMA_AE_AMP_BOUNDS_VIOLATION 0x0108
+#define IRDMA_AE_AMP_RIGHTS_VIOLATION 0x0109
+#define IRDMA_AE_AMP_TO_WRAP 0x010a
+#define IRDMA_AE_AMP_FASTREG_VALID_STAG 0x010c
+#define IRDMA_AE_AMP_FASTREG_MW_STAG 0x010d
+#define IRDMA_AE_AMP_FASTREG_INVALID_RIGHTS 0x010e
+#define IRDMA_AE_AMP_FASTREG_INVALID_LENGTH 0x0110
+#define IRDMA_AE_AMP_INVALIDATE_SHARED 0x0111
+#define IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS 0x0112
+#define IRDMA_AE_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS 0x0113
+#define IRDMA_AE_AMP_MWBIND_VALID_STAG 0x0114
+#define IRDMA_AE_AMP_MWBIND_OF_MR_STAG 0x0115
+#define IRDMA_AE_AMP_MWBIND_TO_ZERO_BASED_STAG 0x0116
+#define IRDMA_AE_AMP_MWBIND_TO_MW_STAG 0x0117
+#define IRDMA_AE_AMP_MWBIND_INVALID_RIGHTS 0x0118
+#define IRDMA_AE_AMP_MWBIND_INVALID_BOUNDS 0x0119
+#define IRDMA_AE_AMP_MWBIND_TO_INVALID_PARENT 0x011a
+#define IRDMA_AE_AMP_MWBIND_BIND_DISABLED 0x011b
+#define IRDMA_AE_PRIV_OPERATION_DENIED 0x011c
+#define IRDMA_AE_AMP_INVALIDATE_TYPE1_MW 0x011d
+#define IRDMA_AE_AMP_MWBIND_ZERO_BASED_TYPE1_MW 0x011e
+#define IRDMA_AE_AMP_FASTREG_INVALID_PBL_HPS_CFG 0x011f
+#define IRDMA_AE_AMP_MWBIND_WRONG_TYPE 0x0120
+#define IRDMA_AE_AMP_FASTREG_PBLE_MISMATCH 0x0121
+#define IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG 0x0132
+#define IRDMA_AE_UDA_XMIT_BAD_PD 0x0133
+#define IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT 0x0134
+#define IRDMA_AE_UDA_L4LEN_INVALID 0x0135
+#define IRDMA_AE_BAD_CLOSE 0x0201
+#define IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE 0x0202
+#define IRDMA_AE_CQ_OPERATION_ERROR 0x0203
+#define IRDMA_AE_RDMA_READ_WHILE_ORD_ZERO 0x0205
+#define IRDMA_AE_STAG_ZERO_INVALID 0x0206
+#define IRDMA_AE_IB_RREQ_AND_Q1_FULL 0x0207
+#define IRDMA_AE_IB_INVALID_REQUEST 0x0208
+#define IRDMA_AE_WQE_UNEXPECTED_OPCODE 0x020a
+#define IRDMA_AE_WQE_INVALID_PARAMETER 0x020b
+#define IRDMA_AE_WQE_INVALID_FRAG_DATA 0x020c
+#define IRDMA_AE_IB_REMOTE_ACCESS_ERROR 0x020d
+#define IRDMA_AE_IB_REMOTE_OP_ERROR 0x020e
+#define IRDMA_AE_WQE_LSMM_TOO_LONG 0x0220
+#define IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN 0x0301
+#define IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER 0x0303
+#define IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION 0x0304
+#define IRDMA_AE_DDP_UBE_INVALID_MO 0x0305
+#define IRDMA_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE 0x0306
+#define IRDMA_AE_DDP_UBE_INVALID_QN 0x0307
+#define IRDMA_AE_DDP_NO_L_BIT 0x0308
+#define IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION 0x0311
+#define IRDMA_AE_RDMAP_ROE_UNEXPECTED_OPCODE 0x0312
+#define IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST 0x0313
+#define IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP 0x0314
+#define IRDMA_AE_ROCE_RSP_LENGTH_ERROR 0x0316
+#define IRDMA_AE_ROCE_EMPTY_MCG 0x0380
+#define IRDMA_AE_ROCE_BAD_MC_IP_ADDR 0x0381
+#define IRDMA_AE_ROCE_BAD_MC_QPID 0x0382
+#define IRDMA_AE_MCG_QP_PROTOCOL_MISMATCH 0x0383
+#define IRDMA_AE_INVALID_ARP_ENTRY 0x0401
+#define IRDMA_AE_INVALID_TCP_OPTION_RCVD 0x0402
+#define IRDMA_AE_STALE_ARP_ENTRY 0x0403
+#define IRDMA_AE_INVALID_AH_ENTRY 0x0406
+#define IRDMA_AE_LLP_CLOSE_COMPLETE 0x0501
+#define IRDMA_AE_LLP_CONNECTION_RESET 0x0502
+#define IRDMA_AE_LLP_FIN_RECEIVED 0x0503
+#define IRDMA_AE_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH 0x0504
+#define IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR 0x0505
+#define IRDMA_AE_LLP_SEGMENT_TOO_SMALL 0x0507
+#define IRDMA_AE_LLP_SYN_RECEIVED 0x0508
+#define IRDMA_AE_LLP_TERMINATE_RECEIVED 0x0509
+#define IRDMA_AE_LLP_TOO_MANY_RETRIES 0x050a
+#define IRDMA_AE_LLP_TOO_MANY_KEEPALIVE_RETRIES 0x050b
+#define IRDMA_AE_LLP_DOUBT_REACHABILITY 0x050c
+#define IRDMA_AE_LLP_CONNECTION_ESTABLISHED 0x050e
+#define IRDMA_AE_RESOURCE_EXHAUSTION 0x0520
+#define IRDMA_AE_RESET_SENT 0x0601
+#define IRDMA_AE_TERMINATE_SENT 0x0602
+#define IRDMA_AE_RESET_NOT_SENT 0x0603
+#define IRDMA_AE_LCE_QP_CATASTROPHIC 0x0700
+#define IRDMA_AE_LCE_FUNCTION_CATASTROPHIC 0x0701
+#define IRDMA_AE_LCE_CQ_CATASTROPHIC 0x0702
+#define IRDMA_AE_QP_SUSPEND_COMPLETE 0x0900
enum irdma_device_caps_const {
IRDMA_WQE_SIZE = 4,
@@ -121,6 +211,7 @@
IRDMA_Q2_BUF_SIZE = 256,
IRDMA_QP_CTX_SIZE = 256,
IRDMA_MAX_PDS = 262144,
+ IRDMA_MIN_WQ_SIZE_GEN2 = 8,
};
enum irdma_addressing_type {
@@ -137,9 +228,15 @@
FLUSH_REM_OP_ERR,
FLUSH_LOC_LEN_ERR,
FLUSH_FATAL_ERR,
+ FLUSH_RETRY_EXC_ERR,
FLUSH_MW_BIND_ERR,
FLUSH_REM_INV_REQ_ERR,
- FLUSH_RETRY_EXC_ERR,
+};
+
+enum irdma_qp_event_type {
+ IRDMA_QP_EVENT_CATASTROPHIC,
+ IRDMA_QP_EVENT_ACCESS_ERR,
+ IRDMA_QP_EVENT_REQ_ERR,
};
enum irdma_cmpl_status {
@@ -195,7 +292,7 @@
struct irdma_ring {
volatile u32 head;
- volatile u32 tail;
+ volatile u32 tail; /* effective tail */
u32 size;
};
@@ -215,14 +312,6 @@
u32 ah_id;
};
-struct irdma_post_inline_send {
- void *data;
- u32 len;
- u32 qkey;
- u32 dest_qp;
- u32 ah_id;
-};
-
struct irdma_post_rq_info {
u64 wr_id;
irdma_sgl sg_list;
@@ -235,12 +324,6 @@
struct irdma_sge rem_addr;
};
-struct irdma_inline_rdma_write {
- void *data;
- u32 len;
- struct irdma_sge rem_addr;
-};
-
struct irdma_rdma_read {
irdma_sgl lo_sg_list;
u32 num_lo_sges;
@@ -283,8 +366,6 @@
struct irdma_rdma_read rdma_read;
struct irdma_bind_window bind_window;
struct irdma_inv_local_stag inv_local_stag;
- struct irdma_inline_rdma_write inline_rdma_write;
- struct irdma_post_inline_send inline_send;
} op;
};
@@ -292,7 +373,6 @@
u64 wr_id;
irdma_qp_handle qp_handle;
u32 bytes_xfered;
- u32 tcp_seq_num_rtt;
u32 qp_id;
u32 ud_src_qpn;
u32 imm_data;
@@ -303,6 +383,7 @@
u16 ud_vlan;
u8 ud_smac[6];
u8 op_type;
+ u8 q_type;
bool stag_invalid_set:1; /* or L_R_Key set */
bool push_dropped:1;
bool error:1;
@@ -312,6 +393,17 @@
bool ud_smac_valid:1;
bool imm_valid:1;
bool signaled:1;
+ union {
+ u32 tcp_sqn;
+ u32 roce_psn;
+ u32 rtt;
+ u32 raw;
+ } stat;
+};
+
+struct qp_err_code {
+ enum irdma_flush_opcode flush_code;
+ enum irdma_qp_event_type event_type;
};
int irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp,
@@ -336,7 +428,7 @@
bool post_sq);
struct irdma_wqe_uk_ops {
- void (*iw_copy_inline_data)(u8 *dest, u8 *src, u32 len, u8 polarity);
+ void (*iw_copy_inline_data)(u8 *dest, struct irdma_sge *sge_list, u32 num_sges, u8 polarity);
u16 (*iw_inline_data_size_to_quanta)(u32 data_size);
void (*iw_set_fragment)(__le64 *wqe, u32 offset, struct irdma_sge *sge,
u8 valid);
@@ -354,6 +446,12 @@
struct irdma_cq_uk_init_info *info);
int irdma_uk_qp_init(struct irdma_qp_uk *qp,
struct irdma_qp_uk_init_info *info);
+void irdma_uk_calc_shift_wq(struct irdma_qp_uk_init_info *ukinfo, u8 *sq_shift,
+ u8 *rq_shift);
+int irdma_uk_calc_depth_shift_sq(struct irdma_qp_uk_init_info *ukinfo,
+ u32 *sq_depth, u8 *sq_shift);
+int irdma_uk_calc_depth_shift_rq(struct irdma_qp_uk_init_info *ukinfo,
+ u32 *rq_depth, u8 *rq_shift);
struct irdma_sq_uk_wr_trk_info {
u64 wrid;
u32 wr_len;
@@ -407,7 +505,6 @@
bool destroy_pending:1; /* Indicates the QP is being destroyed */
void *back_qp;
pthread_spinlock_t *lock;
- bool force_fence;
u8 dbg_rq_flushed;
u16 ord_cnt;
u8 sq_flush_seen;
@@ -444,8 +541,12 @@
u32 max_sq_frag_cnt;
u32 max_rq_frag_cnt;
u32 max_inline_data;
+ u32 sq_depth;
+ u32 rq_depth;
u8 first_sq_wq;
u8 type;
+ u8 sq_shift;
+ u8 rq_shift;
u8 rd_fence_rate;
int abi_ver;
bool legacy_mode;
@@ -462,7 +563,7 @@
};
__le64 *irdma_qp_get_next_send_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx,
- u16 quanta, u32 total_size,
+ u16 *quanta, u32 total_size,
struct irdma_post_sq_info *info);
__le64 *irdma_qp_get_next_recv_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx);
int irdma_uk_clean_cq(void *q, struct irdma_cq_uk *cq);
@@ -471,9 +572,81 @@
int irdma_fragcnt_to_wqesize_rq(u32 frag_cnt, u16 *wqe_size);
void irdma_get_wqe_shift(struct irdma_uk_attrs *uk_attrs, u32 sge,
u32 inline_data, u8 *shift);
-int irdma_get_sqdepth(u32 max_hw_wq_quanta, u32 sq_size, u8 shift, u32 *wqdepth);
-int irdma_get_rqdepth(u32 max_hw_rq_quanta, u32 rq_size, u8 shift, u32 *wqdepth);
+int irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift, u32 *sqdepth);
+int irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift, u32 *rqdepth);
+int irdma_get_srqdepth(struct irdma_uk_attrs *uk_attrs, u32 srq_size, u8 shift, u32 *srqdepth);
void irdma_qp_push_wqe(struct irdma_qp_uk *qp, __le64 *wqe, u16 quanta,
u32 wqe_idx, bool post_sq);
void irdma_clr_wqes(struct irdma_qp_uk *qp, u32 qp_wqe_idx);
+
+static inline struct qp_err_code irdma_ae_to_qp_err_code(u16 ae_id)
+{
+ struct qp_err_code qp_err = { 0 };
+
+ switch (ae_id) {
+ case IRDMA_AE_AMP_BOUNDS_VIOLATION:
+ case IRDMA_AE_AMP_INVALID_STAG:
+ case IRDMA_AE_AMP_RIGHTS_VIOLATION:
+ case IRDMA_AE_AMP_UNALLOCATED_STAG:
+ case IRDMA_AE_AMP_BAD_PD:
+ case IRDMA_AE_AMP_BAD_QP:
+ case IRDMA_AE_AMP_BAD_STAG_KEY:
+ case IRDMA_AE_AMP_BAD_STAG_INDEX:
+ case IRDMA_AE_AMP_TO_WRAP:
+ case IRDMA_AE_PRIV_OPERATION_DENIED:
+ qp_err.flush_code = FLUSH_PROT_ERR;
+ qp_err.event_type = IRDMA_QP_EVENT_ACCESS_ERR;
+ break;
+ case IRDMA_AE_UDA_XMIT_BAD_PD:
+ case IRDMA_AE_WQE_UNEXPECTED_OPCODE:
+ qp_err.flush_code = FLUSH_LOC_QP_OP_ERR;
+ qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
+ break;
+ case IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT:
+ case IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG:
+ case IRDMA_AE_UDA_L4LEN_INVALID:
+ case IRDMA_AE_DDP_UBE_INVALID_MO:
+ case IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
+ qp_err.flush_code = FLUSH_LOC_LEN_ERR;
+ qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
+ break;
+ case IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
+ case IRDMA_AE_IB_REMOTE_ACCESS_ERROR:
+ qp_err.flush_code = FLUSH_REM_ACCESS_ERR;
+ qp_err.event_type = IRDMA_QP_EVENT_ACCESS_ERR;
+ break;
+ case IRDMA_AE_AMP_MWBIND_INVALID_RIGHTS:
+ case IRDMA_AE_AMP_MWBIND_BIND_DISABLED:
+ case IRDMA_AE_AMP_MWBIND_INVALID_BOUNDS:
+ case IRDMA_AE_AMP_MWBIND_VALID_STAG:
+ qp_err.flush_code = FLUSH_MW_BIND_ERR;
+ qp_err.event_type = IRDMA_QP_EVENT_ACCESS_ERR;
+ break;
+ case IRDMA_AE_LLP_TOO_MANY_RETRIES:
+ qp_err.flush_code = FLUSH_RETRY_EXC_ERR;
+ qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
+ break;
+ case IRDMA_AE_IB_INVALID_REQUEST:
+ qp_err.flush_code = FLUSH_REM_INV_REQ_ERR;
+ qp_err.event_type = IRDMA_QP_EVENT_REQ_ERR;
+ break;
+ case IRDMA_AE_LLP_SEGMENT_TOO_SMALL:
+ case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR:
+ case IRDMA_AE_ROCE_RSP_LENGTH_ERROR:
+ case IRDMA_AE_IB_REMOTE_OP_ERROR:
+ qp_err.flush_code = FLUSH_REM_OP_ERR;
+ qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
+ break;
+ case IRDMA_AE_LCE_QP_CATASTROPHIC:
+ qp_err.flush_code = FLUSH_FATAL_ERR;
+ qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
+ break;
+ default:
+ qp_err.flush_code = FLUSH_GENERAL_ERR;
+ qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
+ break;
+ }
+
+ return qp_err;
+}
#endif /* IRDMA_USER_H */
diff --git a/contrib/ofed/libirdma/irdma_uverbs.c b/contrib/ofed/libirdma/irdma_uverbs.c
--- a/contrib/ofed/libirdma/irdma_uverbs.c
+++ b/contrib/ofed/libirdma/irdma_uverbs.c
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (C) 2019 - 2021 Intel Corporation
+ * Copyright (C) 2019 - 2022 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -46,6 +46,7 @@
#include <sys/stat.h>
#include <fcntl.h>
#include <stdbool.h>
+#include <infiniband/opcode.h>
#include "irdma_umain.h"
#include "abi.h"
@@ -137,7 +138,7 @@
struct irdma_upd *iwupd;
int err;
- iwupd = malloc(sizeof(*iwupd));
+ iwupd = calloc(1, sizeof(*iwupd));
if (!iwupd)
return NULL;
@@ -163,6 +164,7 @@
int
irdma_ufree_pd(struct ibv_pd *pd)
{
+ struct irdma_uvcontext *iwvctx = container_of(pd->context, struct irdma_uvcontext, ibv_ctx);
struct irdma_upd *iwupd;
int ret;
@@ -188,27 +190,44 @@
irdma_ureg_mr(struct ibv_pd *pd, void *addr, size_t length,
int access)
{
- struct irdma_umr *umr;
- struct irdma_ureg_mr cmd;
+ struct verbs_mr *vmr;
+ struct irdma_ureg_mr cmd = {};
struct ibv_reg_mr_resp resp;
int err;
- umr = malloc(sizeof(*umr));
- if (!umr)
+ vmr = malloc(sizeof(*vmr));
+ if (!vmr)
return NULL;
cmd.reg_type = IRDMA_MEMREG_TYPE_MEM;
err = ibv_cmd_reg_mr(pd, addr, length,
- (uintptr_t)addr, access, &umr->vmr.ibv_mr, &cmd.ibv_cmd,
+ (uintptr_t)addr, access, &vmr->ibv_mr, &cmd.ibv_cmd,
sizeof(cmd), &resp, sizeof(resp));
if (err) {
- free(umr);
+ free(vmr);
errno = err;
return NULL;
}
- umr->acc_flags = access;
- return &umr->vmr.ibv_mr;
+ return &vmr->ibv_mr;
+}
+
+/*
+ * irdma_urereg_mr - re-register memory region @vmr: mr that was allocated @flags: bit mask to indicate which of the
+ * attr's of MR modified @pd: pd of the mr @addr: user address of the memory region @length: length of the memory
+ * @access: access allowed on this mr
+ */
+int
+irdma_urereg_mr(struct verbs_mr *vmr, int flags, struct ibv_pd *pd,
+ void *addr, size_t length, int access)
+{
+ struct irdma_urereg_mr cmd = {};
+ struct ibv_rereg_mr_resp resp;
+
+ cmd.reg_type = IRDMA_MEMREG_TYPE_MEM;
+ return ibv_cmd_rereg_mr(&vmr->ibv_mr, flags, addr, length, (uintptr_t)addr,
+ access, pd, &cmd.ibv_cmd, sizeof(cmd), &resp,
+ sizeof(resp));
}
/**
@@ -218,19 +237,15 @@
int
irdma_udereg_mr(struct ibv_mr *mr)
{
- struct irdma_umr *umr;
struct verbs_mr *vmr;
int ret;
vmr = container_of(mr, struct verbs_mr, ibv_mr);
- umr = container_of(vmr, struct irdma_umr, vmr);
ret = ibv_cmd_dereg_mr(mr);
if (ret)
return ret;
- free(umr);
-
return 0;
}
@@ -245,6 +260,7 @@
struct ibv_mw *mw;
struct ibv_alloc_mw cmd;
struct ibv_alloc_mw_resp resp;
+ int err;
mw = calloc(1, sizeof(*mw));
if (!mw)
@@ -273,7 +289,6 @@
{
struct ibv_mw_bind_info *bind_info = &mw_bind->bind_info;
struct verbs_mr *vmr;
- struct irdma_umr *umr;
struct ibv_send_wr wr = {};
struct ibv_send_wr *bad_wr;
@@ -284,11 +299,10 @@
if (bind_info->mr) {
vmr = verbs_get_mr(bind_info->mr);
- umr = container_of(vmr, struct irdma_umr, vmr);
if (vmr->mr_type != IBV_MR_TYPE_MR)
return ENOTSUP;
- if (umr->acc_flags & IBV_ACCESS_ZERO_BASED)
+ if (vmr->access & IBV_ACCESS_ZERO_BASED)
return EINVAL;
if (mw->pd != bind_info->mr->pd)
@@ -356,14 +370,15 @@
* get_cq_size - returns actual cqe needed by HW
* @ncqe: minimum cqes requested by application
* @hw_rev: HW generation
+ * @cqe_64byte_ena: enable 64byte cqe
*/
static inline int
-get_cq_size(int ncqe, u8 hw_rev)
+get_cq_size(int ncqe, u8 hw_rev, bool cqe_64byte_ena)
{
ncqe++;
/* Completions with immediate require 1 extra entry */
- if (hw_rev > IRDMA_GEN_1)
+ if (!cqe_64byte_ena && hw_rev > IRDMA_GEN_1)
ncqe *= 2;
if (ncqe < IRDMA_U_MINCQ_SIZE)
@@ -372,8 +387,11 @@
return ncqe;
}
-static inline size_t get_cq_total_bytes(u32 cq_size) {
- return roundup(cq_size * sizeof(struct irdma_cqe), IRDMA_HW_PAGE_SIZE);
+static inline size_t get_cq_total_bytes(u32 cq_size, bool cqe_64byte_ena){
+ if (cqe_64byte_ena)
+ return roundup(cq_size * sizeof(struct irdma_extended_cqe), IRDMA_HW_PAGE_SIZE);
+ else
+ return roundup(cq_size * sizeof(struct irdma_cqe), IRDMA_HW_PAGE_SIZE);
}
/**
@@ -401,17 +419,22 @@
u32 cq_pages;
int ret, ncqe;
u8 hw_rev;
+ bool cqe_64byte_ena;
iwvctx = container_of(context, struct irdma_uvcontext, ibv_ctx);
uk_attrs = &iwvctx->uk_attrs;
hw_rev = uk_attrs->hw_rev;
- if (ext_cq && hw_rev == IRDMA_GEN_1) {
- errno = EOPNOTSUPP;
- return NULL;
+ if (ext_cq) {
+ u32 supported_flags = IRDMA_STANDARD_WC_FLAGS_EX;
+
+ if (hw_rev == IRDMA_GEN_1 || attr_ex->wc_flags & ~supported_flags) {
+ errno = EOPNOTSUPP;
+ return NULL;
+ }
}
- if (attr_ex->cqe < IRDMA_MIN_CQ_SIZE || attr_ex->cqe > uk_attrs->max_hw_cq_size) {
+ if (attr_ex->cqe < uk_attrs->min_hw_cq_size || attr_ex->cqe > uk_attrs->max_hw_cq_size - 1) {
errno = EINVAL;
return NULL;
}
@@ -428,11 +451,12 @@
return NULL;
}
- info.cq_size = get_cq_size(attr_ex->cqe, hw_rev);
+ cqe_64byte_ena = uk_attrs->feature_flags & IRDMA_FEATURE_64_BYTE_CQE ? true : false;
+ info.cq_size = get_cq_size(attr_ex->cqe, hw_rev, cqe_64byte_ena);
iwucq->comp_vector = attr_ex->comp_vector;
LIST_INIT(&iwucq->resize_list);
LIST_INIT(&iwucq->cmpl_generated);
- total_size = get_cq_total_bytes(info.cq_size);
+ total_size = get_cq_total_bytes(info.cq_size, cqe_64byte_ena);
cq_pages = total_size >> IRDMA_HW_PAGE_SHIFT;
if (!(uk_attrs->feature_flags & IRDMA_FEATURE_CQ_RESIZE))
@@ -462,7 +486,7 @@
if (uk_attrs->feature_flags & IRDMA_FEATURE_CQ_RESIZE) {
info.shadow_area = irdma_alloc_hw_buf(IRDMA_DB_SHADOW_AREA_SIZE);
if (!info.shadow_area)
- goto err_dereg_mr;
+ goto err_alloc_shadow;
memset(info.shadow_area, 0, IRDMA_DB_SHADOW_AREA_SIZE);
reg_mr_shadow_cmd.reg_type = IRDMA_MEMREG_TYPE_CQ;
@@ -474,8 +498,9 @@
&reg_mr_shadow_cmd.ibv_cmd, sizeof(reg_mr_shadow_cmd),
&reg_mr_shadow_resp, sizeof(reg_mr_shadow_resp));
if (ret) {
+ irdma_free_hw_buf(info.shadow_area, IRDMA_DB_SHADOW_AREA_SIZE);
errno = ret;
- goto err_dereg_shadow;
+ goto err_alloc_shadow;
}
iwucq->vmr_shadow_area.ibv_mr.pd = &iwvctx->iwupd->ibv_pd;
@@ -491,28 +516,30 @@
ret = ibv_cmd_create_cq_ex(context, attr_ex, &iwucq->verbs_cq.cq_ex,
&cmd.ibv_cmd, sizeof(cmd.ibv_cmd), sizeof(cmd), &resp.ibv_resp,
sizeof(resp.ibv_resp), sizeof(resp));
+ attr_ex->cqe = ncqe;
if (ret) {
errno = ret;
- goto err_dereg_shadow;
+ goto err_create_cq;
}
if (ext_cq)
irdma_ibvcq_ex_fill_priv_funcs(iwucq, attr_ex);
info.cq_id = resp.cq_id;
- /* Do not report the cqe's burned by HW */
+ /* Do not report the CQE's reserved for immediate and burned by HW */
iwucq->verbs_cq.cq.cqe = ncqe;
-
+ if (cqe_64byte_ena)
+ info.avoid_mem_cflct = true;
info.cqe_alloc_db = (u32 *)((u8 *)iwvctx->db + IRDMA_DB_CQ_OFFSET);
irdma_uk_cq_init(&iwucq->cq, &info);
-
return &iwucq->verbs_cq.cq_ex;
-err_dereg_shadow:
- ibv_cmd_dereg_mr(&iwucq->vmr.ibv_mr);
+err_create_cq:
if (iwucq->vmr_shadow_area.ibv_mr.handle) {
ibv_cmd_dereg_mr(&iwucq->vmr_shadow_area.ibv_mr);
- irdma_free_hw_buf(info.shadow_area, IRDMA_HW_PAGE_SIZE);
+ irdma_free_hw_buf(info.shadow_area, IRDMA_DB_SHADOW_AREA_SIZE);
}
+err_alloc_shadow:
+ ibv_cmd_dereg_mr(&iwucq->vmr.ibv_mr);
err_dereg_mr:
irdma_free_hw_buf(info.cq_base, total_size);
err_cq_base:
@@ -545,11 +572,6 @@
irdma_ucreate_cq_ex(struct ibv_context *context,
struct ibv_cq_init_attr_ex *attr_ex)
{
- if (attr_ex->wc_flags & ~IRDMA_CQ_SUPPORTED_WC_FLAGS) {
- errno = EOPNOTSUPP;
- return NULL;
- }
-
return ucreate_cq(context, attr_ex, true);
}
@@ -561,7 +583,7 @@
irdma_free_cq_buf(struct irdma_cq_buf *cq_buf)
{
ibv_cmd_dereg_mr(&cq_buf->vmr.ibv_mr);
- irdma_free_hw_buf(cq_buf->cq.cq_base, get_cq_total_bytes(cq_buf->cq.cq_size));
+ irdma_free_hw_buf(cq_buf->cq.cq_base, cq_buf->buf_size);
free(cq_buf);
}
@@ -645,7 +667,7 @@
ukcq = &iwucq->cq;
cqe = IRDMA_GET_CURRENT_CQ_ELEM(ukcq);
get_64bit_val(cqe, 24, &qword3);
- polarity = (__u8) RS_64(qword3, IRDMA_CQ_VALID);
+ polarity = (__u8) FIELD_GET(IRDMA_CQ_VALID, qword3);
return polarity != ukcq->polarity;
}
@@ -680,7 +702,7 @@
cmpl->cpi.wr_id = qp->sq_wrtrk_array[wqe_idx].wrid;
sw_wqe = qp->sq_base[wqe_idx].elem;
get_64bit_val(sw_wqe, 24, &wqe_qword);
- cmpl->cpi.op_type = (__u8) RS_64(wqe_qword, IRDMAQPSQ_OPCODE);
+ cmpl->cpi.op_type = (__u8) FIELD_GET(IRDMAQPSQ_OPCODE, wqe_qword);
/* remove the SQ WR by moving SQ tail */
IRDMA_RING_SET_TAIL(*sq_ring, sq_ring->tail + qp->sq_wrtrk_array[sq_ring->tail].quanta);
LIST_INSERT_HEAD(&iwuqp->send_cq->cmpl_generated, cmpl, list);
@@ -794,6 +816,55 @@
}
}
+static inline void
+set_ib_wc_op_sq(struct irdma_cq_poll_info *cur_cqe, struct ibv_wc *entry)
+{
+ switch (cur_cqe->op_type) {
+ case IRDMA_OP_TYPE_RDMA_WRITE:
+ case IRDMA_OP_TYPE_RDMA_WRITE_SOL:
+ entry->opcode = IBV_WC_RDMA_WRITE;
+ break;
+ case IRDMA_OP_TYPE_RDMA_READ:
+ entry->opcode = IBV_WC_RDMA_READ;
+ break;
+ case IRDMA_OP_TYPE_SEND_SOL:
+ case IRDMA_OP_TYPE_SEND_SOL_INV:
+ case IRDMA_OP_TYPE_SEND_INV:
+ case IRDMA_OP_TYPE_SEND:
+ entry->opcode = IBV_WC_SEND;
+ break;
+ case IRDMA_OP_TYPE_BIND_MW:
+ entry->opcode = IBV_WC_BIND_MW;
+ break;
+ case IRDMA_OP_TYPE_INV_STAG:
+ entry->opcode = IBV_WC_LOCAL_INV;
+ break;
+ default:
+ entry->status = IBV_WC_GENERAL_ERR;
+ printf("%s: Invalid opcode = %d in CQE\n",
+ __func__, cur_cqe->op_type);
+ }
+}
+
+static inline void
+set_ib_wc_op_rq(struct irdma_cq_poll_info *cur_cqe,
+ struct ibv_wc *entry, bool send_imm_support)
+{
+ if (!send_imm_support) {
+ entry->opcode = cur_cqe->imm_valid ? IBV_WC_RECV_RDMA_WITH_IMM :
+ IBV_WC_RECV;
+ return;
+ }
+ switch (cur_cqe->op_type) {
+ case IBV_OPCODE_RDMA_WRITE_ONLY_WITH_IMMEDIATE:
+ case IBV_OPCODE_RDMA_WRITE_LAST_WITH_IMMEDIATE:
+ entry->opcode = IBV_WC_RECV_RDMA_WITH_IMM;
+ break;
+ default:
+ entry->opcode = IBV_WC_RECV;
+ }
+}
+
/**
* irdma_process_cqe_ext - process current cqe for extended CQ
* @cur_cqe - current cqe info
@@ -830,9 +901,8 @@
ib_qp = qp->back_qp;
if (cur_cqe->error) {
- if (cur_cqe->comp_status == IRDMA_COMPL_STATUS_FLUSHED)
- entry->status = (cur_cqe->comp_status == IRDMA_COMPL_STATUS_FLUSHED) ?
- irdma_flush_err_to_ib_wc_status(cur_cqe->minor_err) : IBV_WC_GENERAL_ERR;
+ entry->status = (cur_cqe->comp_status == IRDMA_COMPL_STATUS_FLUSHED) ?
+ irdma_flush_err_to_ib_wc_status(cur_cqe->minor_err) : IBV_WC_GENERAL_ERR;
entry->vendor_err = cur_cqe->major_err << 16 |
cur_cqe->minor_err;
} else {
@@ -844,47 +914,17 @@
entry->wc_flags |= IBV_WC_WITH_IMM;
}
- switch (cur_cqe->op_type) {
- case IRDMA_OP_TYPE_RDMA_WRITE:
- case IRDMA_OP_TYPE_RDMA_WRITE_SOL:
- entry->opcode = IBV_WC_RDMA_WRITE;
- break;
- case IRDMA_OP_TYPE_RDMA_READ:
- entry->opcode = IBV_WC_RDMA_READ;
- break;
- case IRDMA_OP_TYPE_SEND_SOL:
- case IRDMA_OP_TYPE_SEND_SOL_INV:
- case IRDMA_OP_TYPE_SEND_INV:
- case IRDMA_OP_TYPE_SEND:
- entry->opcode = IBV_WC_SEND;
- break;
- case IRDMA_OP_TYPE_BIND_MW:
- entry->opcode = IBV_WC_BIND_MW;
- break;
- case IRDMA_OP_TYPE_REC:
- entry->opcode = IBV_WC_RECV;
- if (ib_qp->qp_type != IBV_QPT_UD &&
- cur_cqe->stag_invalid_set) {
- entry->invalidated_rkey = cur_cqe->inv_stag;
- entry->wc_flags |= IBV_WC_WITH_INV;
- }
- break;
- case IRDMA_OP_TYPE_REC_IMM:
- entry->opcode = IBV_WC_RECV_RDMA_WITH_IMM;
+ if (cur_cqe->q_type == IRDMA_CQE_QTYPE_SQ) {
+ set_ib_wc_op_sq(cur_cqe, entry);
+ } else {
+ set_ib_wc_op_rq(cur_cqe, entry,
+ qp->qp_caps & IRDMA_SEND_WITH_IMM ?
+ true : false);
if (ib_qp->qp_type != IBV_QPT_UD &&
cur_cqe->stag_invalid_set) {
entry->invalidated_rkey = cur_cqe->inv_stag;
entry->wc_flags |= IBV_WC_WITH_INV;
}
- break;
- case IRDMA_OP_TYPE_INV_STAG:
- entry->opcode = IBV_WC_LOCAL_INV;
- break;
- default:
- entry->status = IBV_WC_GENERAL_ERR;
- printf("%s: Invalid opcode = %d in CQE\n",
- __func__, cur_cqe->op_type);
- return;
}
if (ib_qp->qp_type == IBV_QPT_UD) {
@@ -1111,20 +1151,6 @@
pthread_spin_unlock(&iwucq->lock);
}
-/**
- * irdma_wc_read_completion_ts - Get completion timestamp
- * @ibvcq_ex: ibv extended CQ
- *
- * Get completion timestamp in HCA clock units
- */
-static uint64_t irdma_wc_read_completion_ts(struct ibv_cq_ex *ibvcq_ex){
- struct irdma_ucq *iwucq = container_of(ibvcq_ex, struct irdma_ucq,
- verbs_cq.cq_ex);
-#define HCA_CORE_CLOCK_800_MHZ 800
-
- return iwucq->cur_cqe.tcp_seq_num_rtt / HCA_CORE_CLOCK_800_MHZ;
-}
-
static enum ibv_wc_opcode
irdma_wc_read_opcode(struct ibv_cq_ex *ibvcq_ex)
{
@@ -1255,11 +1281,6 @@
ibvcq_ex->end_poll = irdma_end_poll;
ibvcq_ex->next_poll = irdma_next_poll;
- if (attr_ex->wc_flags & IBV_WC_EX_WITH_COMPLETION_TIMESTAMP) {
- ibvcq_ex->read_completion_ts = irdma_wc_read_completion_ts;
- iwucq->report_rtt = true;
- }
-
ibvcq_ex->read_opcode = irdma_wc_read_opcode;
ibvcq_ex->read_vendor_err = irdma_wc_read_vendor_err;
ibvcq_ex->read_wc_flags = irdma_wc_read_wc_flags;
@@ -1403,15 +1424,13 @@
* @pd: pd for the qp
* @attr: attributes of qp passed
* @resp: response back from create qp
- * @sqdepth: depth of sq
- * @rqdepth: depth of rq
- * @info: info for initializing user level qp
+ * @info: uk info for initializing user level qp
* @abi_ver: abi version of the create qp command
*/
static int
irdma_vmapped_qp(struct irdma_uqp *iwuqp, struct ibv_pd *pd,
- struct ibv_qp_init_attr *attr, int sqdepth,
- int rqdepth, struct irdma_qp_uk_init_info *info,
+ struct ibv_qp_init_attr *attr,
+ struct irdma_qp_uk_init_info *info,
bool legacy_mode)
{
struct irdma_ucreate_qp cmd = {};
@@ -1421,8 +1440,8 @@
struct ibv_reg_mr_resp reg_mr_resp = {};
int ret;
- sqsize = roundup(sqdepth * IRDMA_QP_WQE_MIN_SIZE, IRDMA_HW_PAGE_SIZE);
- rqsize = roundup(rqdepth * IRDMA_QP_WQE_MIN_SIZE, IRDMA_HW_PAGE_SIZE);
+ sqsize = roundup(info->sq_depth * IRDMA_QP_WQE_MIN_SIZE, IRDMA_HW_PAGE_SIZE);
+ rqsize = roundup(info->rq_depth * IRDMA_QP_WQE_MIN_SIZE, IRDMA_HW_PAGE_SIZE);
totalqpsize = rqsize + sqsize + IRDMA_DB_SHADOW_AREA_SIZE;
info->sq = irdma_alloc_hw_buf(totalqpsize);
iwuqp->buf_size = totalqpsize;
@@ -1491,8 +1510,6 @@
struct irdma_uk_attrs *uk_attrs;
struct irdma_uvcontext *iwvctx;
struct irdma_uqp *iwuqp;
- u32 sqdepth, rqdepth;
- u8 sqshift, rqshift;
int status;
if (attr->qp_type != IBV_QPT_RC && attr->qp_type != IBV_QPT_UD) {
@@ -1512,12 +1529,15 @@
return NULL;
}
- irdma_get_wqe_shift(uk_attrs,
- uk_attrs->hw_rev > IRDMA_GEN_1 ? attr->cap.max_send_sge + 1 :
- attr->cap.max_send_sge,
- attr->cap.max_inline_data, &sqshift);
- status = irdma_get_sqdepth(uk_attrs->max_hw_wq_quanta,
- attr->cap.max_send_wr, sqshift, &sqdepth);
+ info.uk_attrs = uk_attrs;
+ info.sq_size = attr->cap.max_send_wr;
+ info.rq_size = attr->cap.max_recv_wr;
+ info.max_sq_frag_cnt = attr->cap.max_send_sge;
+ info.max_rq_frag_cnt = attr->cap.max_recv_sge;
+ info.max_inline_data = attr->cap.max_inline_data;
+ info.abi_ver = iwvctx->abi_ver;
+
+ status = irdma_uk_calc_depth_shift_sq(&info, &info.sq_depth, &info.sq_shift);
if (status) {
printf("%s: invalid SQ attributes, max_send_wr=%d max_send_sge=%d max_inline=%d\n",
__func__, attr->cap.max_send_wr, attr->cap.max_send_sge,
@@ -1526,14 +1546,7 @@
return NULL;
}
- if (uk_attrs->hw_rev == IRDMA_GEN_1 && iwvctx->abi_ver > 4)
- rqshift = IRDMA_MAX_RQ_WQE_SHIFT_GEN1;
- else
- irdma_get_wqe_shift(uk_attrs, attr->cap.max_recv_sge, 0,
- &rqshift);
-
- status = irdma_get_rqdepth(uk_attrs->max_hw_rq_quanta,
- attr->cap.max_recv_wr, rqshift, &rqdepth);
+ status = irdma_uk_calc_depth_shift_rq(&info, &info.rq_depth, &info.rq_shift);
if (status) {
printf("%s: invalid RQ attributes, recv_wr=%d recv_sge=%d\n",
__func__, attr->cap.max_recv_wr, attr->cap.max_recv_sge);
@@ -1550,31 +1563,35 @@
if (pthread_spin_init(&iwuqp->lock, PTHREAD_PROCESS_PRIVATE))
goto err_free_qp;
- info.sq_size = sqdepth >> sqshift;
- info.rq_size = rqdepth >> rqshift;
- attr->cap.max_send_wr = info.sq_size;
- attr->cap.max_recv_wr = info.rq_size;
+ info.sq_size = info.sq_depth >> info.sq_shift;
+ info.rq_size = info.rq_depth >> info.rq_shift;
+ /**
+ * For older ABI version (less than 6) passes raw sq and rq
+ * quanta in cap.max_send_wr and cap.max_recv_wr.
+ * But then kernel had no way of calculating the actual qp size.
+ */
+ if (iwvctx->abi_ver <= 5) {
+ attr->cap.max_send_wr = info.sq_size;
+ attr->cap.max_recv_wr = info.rq_size;
+ }
- info.uk_attrs = uk_attrs;
- info.max_sq_frag_cnt = attr->cap.max_send_sge;
- info.max_rq_frag_cnt = attr->cap.max_recv_sge;
iwuqp->recv_sges = calloc(attr->cap.max_recv_sge, sizeof(*iwuqp->recv_sges));
if (!iwuqp->recv_sges)
goto err_destroy_lock;
info.wqe_alloc_db = (u32 *)iwvctx->db;
info.legacy_mode = iwvctx->legacy_mode;
- info.sq_wrtrk_array = calloc(sqdepth, sizeof(*info.sq_wrtrk_array));
+ info.sq_wrtrk_array = calloc(info.sq_depth, sizeof(*info.sq_wrtrk_array));
if (!info.sq_wrtrk_array)
goto err_free_rsges;
- info.rq_wrid_array = calloc(rqdepth, sizeof(*info.rq_wrid_array));
+ info.rq_wrid_array = calloc(info.rq_depth, sizeof(*info.rq_wrid_array));
if (!info.rq_wrid_array)
goto err_free_sq_wrtrk;
iwuqp->sq_sig_all = attr->sq_sig_all;
iwuqp->qp_type = attr->qp_type;
- status = irdma_vmapped_qp(iwuqp, pd, attr, sqdepth, rqdepth, &info, iwvctx->legacy_mode);
+ status = irdma_vmapped_qp(iwuqp, pd, attr, &info, iwvctx->legacy_mode);
if (status) {
errno = status;
goto err_free_rq_wrid;
@@ -1583,18 +1600,15 @@
iwuqp->qp.back_qp = iwuqp;
iwuqp->qp.lock = &iwuqp->lock;
- info.max_sq_frag_cnt = attr->cap.max_send_sge;
- info.max_rq_frag_cnt = attr->cap.max_recv_sge;
- info.max_inline_data = attr->cap.max_inline_data;
- iwuqp->qp.force_fence = true;
status = irdma_uk_qp_init(&iwuqp->qp, &info);
if (status) {
errno = status;
goto err_free_vmap_qp;
}
- attr->cap.max_send_wr = (sqdepth - IRDMA_SQ_RSVD) >> sqshift;
- attr->cap.max_recv_wr = (rqdepth - IRDMA_RQ_RSVD) >> rqshift;
+ attr->cap.max_send_wr = (info.sq_depth - IRDMA_SQ_RSVD) >> info.sq_shift;
+ attr->cap.max_recv_wr = (info.rq_depth - IRDMA_RQ_RSVD) >> info.rq_shift;
+
return &iwuqp->ibv_qp;
err_free_vmap_qp:
@@ -1649,8 +1663,6 @@
iwuqp = container_of(qp, struct irdma_uqp, ibv_qp);
iwvctx = container_of(qp->context, struct irdma_uvcontext, ibv_ctx);
- iwuqp->attr_mask = attr_mask;
- memcpy(&iwuqp->attr, attr, sizeof(iwuqp->attr));
if (iwuqp->qp.qp_caps & IRDMA_PUSH_MODE && attr_mask & IBV_QP_STATE &&
iwvctx->uk_attrs.hw_rev > IRDMA_GEN_1) {
@@ -1707,13 +1719,13 @@
{
struct irdma_umodify_qp_resp resp = {};
struct irdma_modify_qp_cmd cmd_ex = {};
- struct irdma_uqp *iwuqp;
+ struct ibv_qp_attr attr = {};
+ attr.qp_state = IBV_QPS_ERR;
cmd_ex.sq_flush = sq_flush;
cmd_ex.rq_flush = rq_flush;
- iwuqp = container_of(qp, struct irdma_uqp, ibv_qp);
- ibv_cmd_modify_qp_ex(qp, &iwuqp->attr, iwuqp->attr_mask,
+ ibv_cmd_modify_qp_ex(qp, &attr, IBV_QP_STATE,
&cmd_ex.ibv_cmd,
sizeof(cmd_ex.ibv_cmd),
sizeof(cmd_ex), &resp.ibv_resp,
@@ -1857,8 +1869,6 @@
info.signaled = true;
if (ib_wr->send_flags & IBV_SEND_FENCE)
info.read_fence = true;
- if (iwuqp->send_cq->report_rtt)
- info.report_rtt = true;
switch (ib_wr->opcode) {
case IBV_WR_SEND_WITH_IMM:
@@ -1885,31 +1895,21 @@
info.op_type = IRDMA_OP_TYPE_SEND_INV;
info.stag_to_inv = ib_wr->imm_data;
}
- if (ib_wr->send_flags & IBV_SEND_INLINE) {
- info.op.inline_send.data = (void *)(uintptr_t)ib_wr->sg_list[0].addr;
- info.op.inline_send.len = ib_wr->sg_list[0].length;
- if (ib_qp->qp_type == IBV_QPT_UD) {
- struct irdma_uah *ah = container_of(ib_wr->wr.ud.ah,
- struct irdma_uah, ibv_ah);
-
- info.op.inline_send.ah_id = ah->ah_id;
- info.op.inline_send.qkey = ib_wr->wr.ud.remote_qkey;
- info.op.inline_send.dest_qp = ib_wr->wr.ud.remote_qpn;
- }
+ info.op.send.num_sges = ib_wr->num_sge;
+ info.op.send.sg_list = (struct irdma_sge *)ib_wr->sg_list;
+ if (ib_qp->qp_type == IBV_QPT_UD) {
+ struct irdma_uah *ah = container_of(ib_wr->wr.ud.ah,
+ struct irdma_uah, ibv_ah);
+
+ info.op.send.ah_id = ah->ah_id;
+ info.op.send.qkey = ib_wr->wr.ud.remote_qkey;
+ info.op.send.dest_qp = ib_wr->wr.ud.remote_qpn;
+ }
+
+ if (ib_wr->send_flags & IBV_SEND_INLINE)
err = irdma_uk_inline_send(&iwuqp->qp, &info, false);
- } else {
- info.op.send.num_sges = ib_wr->num_sge;
- info.op.send.sg_list = (struct irdma_sge *)ib_wr->sg_list;
- if (ib_qp->qp_type == IBV_QPT_UD) {
- struct irdma_uah *ah = container_of(ib_wr->wr.ud.ah,
- struct irdma_uah, ibv_ah);
-
- info.op.inline_send.ah_id = ah->ah_id;
- info.op.inline_send.qkey = ib_wr->wr.ud.remote_qkey;
- info.op.inline_send.dest_qp = ib_wr->wr.ud.remote_qpn;
- }
+ else
err = irdma_uk_send(&iwuqp->qp, &info, false);
- }
break;
case IBV_WR_RDMA_WRITE_WITH_IMM:
if (iwuqp->qp.qp_caps & IRDMA_WRITE_WITH_IMM) {
@@ -1926,19 +1926,14 @@
else
info.op_type = IRDMA_OP_TYPE_RDMA_WRITE;
- if (ib_wr->send_flags & IBV_SEND_INLINE) {
- info.op.inline_rdma_write.data = (void *)(uintptr_t)ib_wr->sg_list[0].addr;
- info.op.inline_rdma_write.len = ib_wr->sg_list[0].length;
- info.op.inline_rdma_write.rem_addr.tag_off = ib_wr->wr.rdma.remote_addr;
- info.op.inline_rdma_write.rem_addr.stag = ib_wr->wr.rdma.rkey;
+ info.op.rdma_write.num_lo_sges = ib_wr->num_sge;
+ info.op.rdma_write.lo_sg_list = (void *)ib_wr->sg_list;
+ info.op.rdma_write.rem_addr.tag_off = ib_wr->wr.rdma.remote_addr;
+ info.op.rdma_write.rem_addr.stag = ib_wr->wr.rdma.rkey;
+ if (ib_wr->send_flags & IBV_SEND_INLINE)
err = irdma_uk_inline_rdma_write(&iwuqp->qp, &info, false);
- } else {
- info.op.rdma_write.lo_sg_list = (void *)ib_wr->sg_list;
- info.op.rdma_write.num_lo_sges = ib_wr->num_sge;
- info.op.rdma_write.rem_addr.tag_off = ib_wr->wr.rdma.remote_addr;
- info.op.rdma_write.rem_addr.stag = ib_wr->wr.rdma.rkey;
+ else
err = irdma_uk_rdma_write(&iwuqp->qp, &info, false);
- }
break;
case IBV_WR_RDMA_READ:
if (ib_wr->num_sge > uk_attrs->max_hw_read_sges) {
@@ -1965,9 +1960,8 @@
info.op.bind_window.mw_stag = ib_wr->bind_mw.rkey;
} else {
struct verbs_mr *vmr = verbs_get_mr(ib_wr->bind_mw.bind_info.mr);
- struct irdma_umr *umr = container_of(vmr, struct irdma_umr, vmr);
- if (umr->acc_flags & IBV_ACCESS_ZERO_BASED) {
+ if (vmr->access & IBV_ACCESS_ZERO_BASED) {
err = EINVAL;
break;
}
@@ -2085,7 +2079,7 @@
{
struct irdma_uah *ah;
union ibv_gid sgid;
- struct irdma_ucreate_ah_resp resp;
+ struct irdma_ucreate_ah_resp resp = {};
int err;
err = ibv_query_gid(ibpd->context, attr->port_num, attr->grh.sgid_index,
@@ -2182,6 +2176,7 @@
u32 cq_pages;
int cqe_needed;
int ret = 0;
+ bool cqe_64byte_ena;
iwucq = container_of(cq, struct irdma_ucq, verbs_cq.cq);
iwvctx = container_of(cq->context, struct irdma_uvcontext, ibv_ctx);
@@ -2190,20 +2185,17 @@
if (!(uk_attrs->feature_flags & IRDMA_FEATURE_CQ_RESIZE))
return EOPNOTSUPP;
- if (cqe > IRDMA_MAX_CQ_SIZE)
+ if (cqe < uk_attrs->min_hw_cq_size || cqe > uk_attrs->max_hw_cq_size - 1)
return EINVAL;
- cqe_needed = cqe + 1;
- if (uk_attrs->hw_rev > IRDMA_GEN_1)
- cqe_needed *= 2;
+ cqe_64byte_ena = uk_attrs->feature_flags & IRDMA_FEATURE_64_BYTE_CQE ? true : false;
- if (cqe_needed < IRDMA_U_MINCQ_SIZE)
- cqe_needed = IRDMA_U_MINCQ_SIZE;
+ cqe_needed = get_cq_size(cqe, uk_attrs->hw_rev, cqe_64byte_ena);
if (cqe_needed == iwucq->cq.cq_size)
return 0;
- cq_size = get_cq_total_bytes(cqe_needed);
+ cq_size = get_cq_total_bytes(cqe_needed, cqe_64byte_ena);
cq_pages = cq_size >> IRDMA_HW_PAGE_SHIFT;
cq_base = irdma_alloc_hw_buf(cq_size);
if (!cq_base)
@@ -2239,6 +2231,7 @@
goto err_resize;
memcpy(&cq_buf->cq, &iwucq->cq, sizeof(cq_buf->cq));
+ cq_buf->buf_size = cq_size;
cq_buf->vmr = iwucq->vmr;
iwucq->vmr = new_mr;
irdma_uk_cq_resize(&iwucq->cq, cq_base, cqe_needed);
diff --git a/contrib/ofed/libirdma/libirdma.map b/contrib/ofed/libirdma/libirdma.map
--- a/contrib/ofed/libirdma/libirdma.map
+++ b/contrib/ofed/libirdma/libirdma.map
@@ -1,10 +1,8 @@
/* Export symbols should be added below according to
Documentation/versioning.md document. */
IRDMA_1.0 {
- global: *;
- local: *;
+ global:
+ libirdma_query_device;
+ local: *;
};
-IRDMA_1.1 {
- global: *;
-} IRDMA_1.0;
diff --git a/contrib/ofed/libirdma/osdep.h b/contrib/ofed/libirdma/osdep.h
--- a/contrib/ofed/libirdma/osdep.h
+++ b/contrib/ofed/libirdma/osdep.h
@@ -91,6 +91,13 @@
#define SPEED_100000 100000
#define BIT_ULL(a) (1ULL << (a))
+#define min(a, b) ((a) > (b) ? (b) : (a))
+#ifndef likely
+#define likely(x) __builtin_expect((x), 1)
+#endif
+#ifndef unlikely
+#define unlikely(x) __builtin_expect((x), 0)
+#endif
#define __aligned_u64 uint64_t __aligned(8)
@@ -131,7 +138,7 @@
#define irdma_dev_warn(a, b, ...) printf(b, ##__VA_ARGS__) /*dev_warn(a, b)*/
#define irdma_dev_info(a, b, ...) printf(b, ##__VA_ARGS__)
#define irdma_pr_warn printf
-#define ibdev_err(ibdev, fmt, ...) dev_err(&((ibdev)->dev), fmt, ##__VA_ARGS__)
+#define ibdev_err(ibdev, fmt, ...) printf("%s:"fmt, (ibdev)->name, ##__VA_ARGS__)
#define dump_struct(s, sz, name) \
do { \
@@ -166,8 +173,13 @@
#define irdma_usec_delay(x) DELAY(x)
#define mdelay(x) DELAY((x) * 1000)
-#define rt_tos2priority(tos) (((tos >> 1) & 0x8 >> 1) | ((tos >> 2) ^ ((tos >> 3) << 1)))
+#define rt_tos2priority(tos) (tos >> 5)
#define ah_attr_to_dmac(attr) ((attr).dmac)
+#define kc_ib_modify_qp_is_ok(cur_state, next_state, type, mask, ll) \
+ ib_modify_qp_is_ok(cur_state, next_state, type, mask)
+#define kc_typeq_ib_wr const
+#define kc_ifp_find ip_ifp_find
+#define kc_ifp6_find ip6_ifp_find
#define kc_rdma_gid_attr_network_type(sgid_attr, gid_type, gid) \
ib_gid_to_network_type(gid_type, gid)
#define irdma_del_timer_compat(tt) del_timer((tt))
@@ -207,6 +219,7 @@
struct verbs_mr {
struct ibv_mr ibv_mr;
enum ibv_mr_type mr_type;
+ int access;
};
#define verbs_get_mr(mr) container_of((mr), struct verbs_mr, ibv_mr)
#endif
diff --git a/share/man/man4/irdma.4 b/share/man/man4/irdma.4
--- a/share/man/man4/irdma.4
+++ b/share/man/man4/irdma.4
@@ -39,8 +39,8 @@
.Nd RDMA FreeBSD driver for Intel(R) Ethernet Controller E810
.Sh SYNOPSIS
This module relies on
-.Xr if_ice 4
-.Bl -tag -nested-width indent
+.Xr ice 4
+.Bl -tag -width indent
.It The following kernel options should be included in the configuration:
.Cd options OFED
.Cd options OFED_DEBUG_INIT
@@ -52,8 +52,9 @@
.Ss Features
The
.Nm
-driver provides RDMA protocol support on RDMA-capable Intel Ethernet 800 Series NICs which are supported by
-.Xr if_ice 4
+driver provides RDMA protocol support on RDMA-capable Intel Ethernet 800 Series
+NICs which are supported by
+.Xr ice 4
.
.Pp
The driver supports both iWARP and RoCEv2 protocols.
@@ -66,48 +67,65 @@
.Bl -tag -width indent
.It Va dev.irdma<interface_number>.roce_enable
enables RoCEv2 protocol usage on <interface_numer> interface.
-.Pp By default RoCEv2 protocol is used.
+.Pp
+By default RoCEv2 protocol is used.
.It Va dev.irdma<interface_number>.dcqcn_cc_cfg_valid
-indicates that all DCQCN parameters are valid and should be updated in registers or QP context.
+indicates that all DCQCN parameters are valid and should be updated in
+registers or QP context.
.Pp
-Setting this parameter to 1 means that settings in
-.Em dcqcn_min_dec_factor, dcqcn_min_rate_MBps, dcqcn_F, dcqcn_T,
-.Em dcqcn_B, dcqcn_rai_factor, dcqcn_hai_factor, dcqcn_rreduce_mperiod
-are taken into account. Otherwise default values are used.
+Setting this parameter to 1 means that settings in
+.Em dcqcn_min_dec_factor , dcqcn_min_rate_MBps , dcqcn_F , dcqcn_T ,
+.Em dcqcn_B, dcqcn_rai_factor, dcqcn_hai_factor, dcqcn_rreduce_mperiod
+are taken into account.
+Otherwise default values are used.
.Pp
Note: "roce_enable" must also be set for this tunable to take effect.
.It Va dev.irdma<interface_number>.dcqcn_min_dec_factor
-The minimum factor by which the current transmit rate can be changed when processing a CNP. Value is given as a percentage (1-100).
+The minimum factor by which the current transmit rate can be changed when
+processing a CNP.
+Value is given as a percentage (1-100).
.Pp
-Note: "roce_enable" and "dcqcn_cc_cfg_valid" must also be set for this tunable to take effect.
+Note: "roce_enable" and "dcqcn_cc_cfg_valid" must also be set for this tunable
+to take effect.
.It Va dev.irdma<interface_number>.dcqcn_min_rate_MBps
The minimum value, in Mbits per second, for rate to limit.
.Pp
-Note: "roce_enable" and "dcqcn_cc_cfg_valid" must also be set for this tunable to take effect.
+Note: "roce_enable" and "dcqcn_cc_cfg_valid" must also be set for this tunable
+to take effect.
.It Va dev.irdma<interface_number>.dcqcn_F
The number of times to stay in each stage of bandwidth recovery.
.Pp
-Note: "roce_enable" and "dcqcn_cc_cfg_valid" must also be set for this tunable to take effect.
+Note: "roce_enable" and "dcqcn_cc_cfg_valid" must also be set for this tunable
+to take effect.
.It Va dev.irdma<interface_number>.dcqcn_T
-The number of microseconds that should elapse before increasing the CWND in DCQCN mode.
+The number of microseconds that should elapse before increasing the CWND
+in DCQCN mode.
.Pp
-Note: "roce_enable" and "dcqcn_cc_cfg_valid" must also be set for this tunable to take effect.
+Note: "roce_enable" and "dcqcn_cc_cfg_valid" must also be set for this tunable
+to take effect.
.It Va dev.irdma<interface_number>.dcqcn_B
The number of bytes to transmit before updating CWND in DCQCN mode.
.Pp
-Note: "roce_enable" and "dcqcn_cc_cfg_valid" must also be set for this tunable to take effect.
+Note: "roce_enable" and "dcqcn_cc_cfg_valid" must also be set for this tunable
+to take effect.
.It Va dev.irdma<interface_number>.dcqcn_rai_factor
The number of MSS to add to the congestion window in additive increase mode.
.Pp
-Note: "roce_enable" and "dcqcn_cc_cfg_valid" must also be set for this tunable to take effect.
+Note: "roce_enable" and "dcqcn_cc_cfg_valid" must also be set for this tunable
+to take effect.
.It Va dev.irdma<interface_number>.dcqcn_hai_factor
The number of MSS to add to the congestion window in hyperactive increase mode.
.Pp
-Note: "roce_enable" and "dcqcn_cc_cfg_valid" must also be set for this tunable to take effect.
+Note: "roce_enable" and "dcqcn_cc_cfg_valid" must also be set for this tunable
+to take effect.
.It Va dev.irdma<interface_number>.dcqcn_rreduce_mperiod
-The minimum time between 2 consecutive rate reductions for a single flow. Rate reduction will occur only if a CNP is received during the relevant time interval.
+The minimum time between 2 consecutive rate reductions for a single flow.
+Rate reduction will occur only if a CNP is received during the relevant time
+interval.
.Pp
-Note: "roce_enable" and "dcqcn_cc_cfg_valid" must also be set for this tunable to take effect.
+Note: "roce_enable" and "dcqcn_cc_cfg_valid" must also be set for this tunable
+to take effect.
+.El
.Ss SYSCTL PROCEDURES
Sysctl controls are available for runtime adjustments.
.Bl -tag -width indent
@@ -120,64 +138,60 @@
.Pp
Note: "roce_enable" must also be set for this sysctl to take effect.
.Pp
-Note: The change may be set at any time, but it will be applied only to newly created QPs.
+Note: The change may be set at any time, but it will be applied only to newly
+created QPs.
+.El
.Ss TESTING
.Bl -enum
.It
To load the irdma driver, run:
-.Bl -tag -width indent
-.It
+.Bd -literal -offset indent
kldload irdma
-.El
-If if_ice is not already loaded, the system will load it on its own. Please check whether the value of sysctl
+.Ed
+If if_ice is not already loaded, the system will load it on its own.
+Please check whether the value of sysctl
.Va hw.ice.irdma
-is 1, if the irdma driver is not loading. To change the value put:
-.Bl -tag -width indent
-.It
+is 1, if the irdma driver is not loading.
+To change the value put:
+.Bd -literal -offset indent
hw.ice.irdma=1
-.El
-to
+.Ed
+in
.Pa /boot/loader.conf
and reboot.
.It
To check that the driver was loaded, run:
-.Bl -tag -width indent
-.It
+.Bd -literal -offset indent
sysctl -a | grep infiniband
-.El
+.Ed
Typically, if everything goes well, around 190 entries per PF will appear.
.It
-Each interface of the card may work in either iWARP or RoCEv2 mode. To enable RoCEv2 compatibility, add:
-.Bl -tag -width indent
-.It
+Each interface of the card may work in either iWARP or RoCEv2 mode.
+To enable RoCEv2 compatibility, add:
+.Bd -literal -offset indent
dev.irdma<interface_number>.roce_enable=1
-.El
+.Ed
where <interface_number> is a desired ice interface number on which
-RoCEv2 protocol needs to be enabled, to:
-.Bl -tag -width indent
-.It
+RoCEv2 protocol needs to be enabled, into:
.Pa /boot/loader.conf
-.El
-for instance:
+, for instance:
.Bl -tag -width indent
-.It
-dev.irdma0.roce_enable=0
-.It
-dev.irdma1.roce_enable=1
+.It dev.irdma0.roce_enable=0
+.It dev.irdma1.roce_enable=1
.El
-will keep iWARP mode on ice0 and enable RoCEv2 mode on interface ice1. The RoCEv2 mode is the default.
-.Dl
+will keep iWARP mode on ice0 and enable RoCEv2 mode on interface ice1.
+The RoCEv2 mode is the default.
+.Pp
To check irdma roce_enable status, run:
-.Bl -tag -width indent
-.It
+.Bd -literal -offset indent
sysctl dev.irdma<interface_number>.roce_enable
-.El
+.Ed
for instance:
-.Bl -tag -width indent
-.It
+.Bd -literal -offset indent
sysctl dev.irdma2.roce_enable
-.El
-with returned value of '0' indicate the iWARP mode, and the value of '1' indicate the RoCEv2 mode.
+.Ed
+with returned value of '0' indicate the iWARP mode, and the value of '1'
+indicate the RoCEv2 mode.
.Pp
Note: An interface configured in one mode will not be able to connect
to a node configured in another mode.
@@ -187,44 +201,42 @@
may lead to significant performance loss or connectivity issues.
.It
Enable flow control in the ice driver:
-.Bl -tag -width indent
-.It
+.Bd -literal -offset indent
sysctl dev.ice.<interface_number>.fc=3
-.El
-Enable flow control on the switch your system is connected to. See your
-switch documentation for details.
+.Ed
+Enable flow control on the switch your system is connected to.
+See your switch documentation for details.
.It
The source code for krping software is provided with the kernel in
-/usr/src/sys/contrib/rdma/krping/. To compile the software, change
-directory to /usr/src/sys/modules/rdma/krping/ and invoke the following:
+/usr/src/sys/contrib/rdma/krping/.
+To compile the software, change directory to
+/usr/src/sys/modules/rdma/krping/ and invoke the following:
.Bl -tag -width indent
-.It
-make clean
-.It
-make
-.It
-make install
+.It make clean
+.It make
+.It make install
+.It kldload krping
.El
.It
Start a krping server on one machine:
-.Bl -tag -width indent
-.It
- echo size=64,count=1,port=6601,addr=100.0.0.189,server > /dev/krping
-.El
+.Bd -literal -offset indent
+echo size=64,count=1,port=6601,addr=100.0.0.189,server > /dev/krping
+.Ed
.It
Connect a client from another machine:
-.Bl -tag -width indent
-.It
- echo size=64,count=1,port=6601,addr=100.0.0.189,client > /dev/krping
+.Bd -literal -offset indent
+echo size=64,count=1,port=6601,addr=100.0.0.189,client > /dev/krping
+.Ed
.El
.Sh SUPPORT
For general information and support, go to the Intel support website at:
.Lk http://support.intel.com/ .
.Pp
-If an issue is identified with this driver with a supported adapter, email all the specific information related to the issue to
+If an issue is identified with this driver with a supported adapter, email all
+the specific information related to the issue to
.Mt freebsd@intel.com .
.Sh SEE ALSO
-.Xr if_ice 4
+.Xr ice 4
.Sh AUTHORS
.An -nosplit
The
diff --git a/sys/dev/irdma/fbsd_kcompat.h b/sys/dev/irdma/fbsd_kcompat.h
--- a/sys/dev/irdma/fbsd_kcompat.h
+++ b/sys/dev/irdma/fbsd_kcompat.h
@@ -53,6 +53,7 @@
BUILD_BUG_ON_ZERO( \
!__same_type(((struct drv_struct *)NULL)->member, \
struct ib_struct)))
+
#define set_ibdev_dma_device(ibdev, dev) \
ibdev.dma_device = (dev)
#define set_max_sge(props, rf) \
@@ -70,7 +71,6 @@
#define IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION IB_CQ_FLAGS_TIMESTAMP_COMPLETION
#define kc_irdma_destroy_qp(ibqp, udata) irdma_destroy_qp(ibqp, udata)
-
#ifndef IB_QP_ATTR_STANDARD_BITS
#define IB_QP_ATTR_STANDARD_BITS GENMASK(20, 0)
#endif
@@ -78,12 +78,15 @@
#define IRDMA_QOS_MODE_VLAN 0x0
#define IRDMA_QOS_MODE_DSCP 0x1
+#define IRDMA_VER_LEN 24
+
void kc_set_roce_uverbs_cmd_mask(struct irdma_device *iwdev);
void kc_set_rdma_uverbs_cmd_mask(struct irdma_device *iwdev);
struct irdma_tunable_info {
struct sysctl_ctx_list irdma_sysctl_ctx;
struct sysctl_oid *irdma_sysctl_tree;
+ char drv_ver[IRDMA_VER_LEN];
u8 roce_ena;
};
@@ -165,6 +168,7 @@
void kc_irdma_put_device(struct irdma_device *iwdev);
void kc_set_loc_seq_num_mss(struct irdma_cm_node *cm_node);
+u16 kc_rdma_get_udp_sport(u32 fl, u32 lqpn, u32 rqpn);
void irdma_get_dev_fw_str(struct ib_device *dev, char *str, size_t str_len);
@@ -182,11 +186,19 @@
u32 irdma_create_stag(struct irdma_device *iwdev);
void irdma_free_stag(struct irdma_device *iwdev, u32 stag);
+int irdma_hwdereg_mr(struct ib_mr *ib_mr);
+int irdma_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, u64 len,
+ u64 virt, int new_access, struct ib_pd *new_pd,
+ struct ib_udata *udata);
struct irdma_mr;
struct irdma_cq;
struct irdma_cq_buf;
struct ib_mr *irdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
u32 max_num_sg, struct ib_udata *udata);
+int irdma_hwreg_mr(struct irdma_device *iwdev, struct irdma_mr *iwmr,
+ u16 access);
+struct ib_mr *irdma_rereg_mr_trans(struct irdma_mr *iwmr, u64 start, u64 len,
+ u64 virt, struct ib_udata *udata);
int irdma_hw_alloc_mw(struct irdma_device *iwdev, struct irdma_mr *iwmr);
struct ib_mw *irdma_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
struct ib_udata *udata);
@@ -201,6 +213,11 @@
struct irdma_qp *iwqp,
struct irdma_qp_init_info *info,
struct ib_qp_init_attr *init_attr);
+int irdma_setup_umode_qp(struct ib_udata *udata,
+ struct irdma_device *iwdev,
+ struct irdma_qp *iwqp,
+ struct irdma_qp_init_info *info,
+ struct ib_qp_init_attr *init_attr);
void irdma_roce_fill_and_set_qpctx_info(struct irdma_qp *iwqp,
struct irdma_qp_host_ctx_info *ctx_info);
void irdma_iw_fill_and_set_qpctx_info(struct irdma_qp *iwqp,
diff --git a/sys/dev/irdma/fbsd_kcompat.c b/sys/dev/irdma/fbsd_kcompat.c
--- a/sys/dev/irdma/fbsd_kcompat.c
+++ b/sys/dev/irdma/fbsd_kcompat.c
@@ -41,6 +41,7 @@
#include <netinet/in_fib.h>
#include <netinet6/in6_fib.h>
#include <net/route/nhop.h>
+#include <net/if_llatbl.h>
/* additional QP debuging option. Keep false unless needed */
bool irdma_upload_context = false;
@@ -389,7 +390,8 @@
if (dst_sin->sa_family == AF_INET) {
err = arpresolve(ifp, gateway, NULL, nexthop, dst_mac, NULL, &lle);
} else if (dst_sin->sa_family == AF_INET6) {
- err = nd6_resolve(ifp, gateway, NULL, nexthop, dst_mac, NULL, &lle);
+ err = nd6_resolve(ifp, LLE_SF(AF_INET6, gateway), NULL, nexthop,
+ dst_mac, NULL, &lle);
} else {
err = -EPROTONOSUPPORT;
}
@@ -467,15 +469,20 @@
irdma_resolve_neigh_lpb_chk(struct irdma_device *iwdev, struct irdma_cm_node *cm_node,
struct irdma_cm_info *cm_info)
{
+#ifdef VIMAGE
struct rdma_cm_id *rdma_id = (struct rdma_cm_id *)cm_node->cm_id->context;
struct vnet *vnet = rdma_id->route.addr.dev_addr.net;
+#endif
int arpindex;
int oldarpindex;
+ bool is_lpb = false;
- if ((cm_node->ipv4 &&
- irdma_ipv4_is_lpb(vnet, cm_node->loc_addr[0], cm_node->rem_addr[0])) ||
- (!cm_node->ipv4 &&
- irdma_ipv6_is_lpb(cm_node->loc_addr, cm_node->rem_addr))) {
+ CURVNET_SET_QUIET(vnet);
+ is_lpb = cm_node->ipv4 ?
+ irdma_ipv4_is_lpb(cm_node->loc_addr[0], cm_node->rem_addr[0]) :
+ irdma_ipv6_is_lpb(cm_node->loc_addr, cm_node->rem_addr);
+ CURVNET_RESTORE();
+ if (is_lpb) {
cm_node->do_lpb = true;
arpindex = irdma_arp_table(iwdev->rf, cm_node->rem_addr,
NULL,
diff --git a/sys/dev/irdma/icrdma.c b/sys/dev/irdma/icrdma.c
--- a/sys/dev/irdma/icrdma.c
+++ b/sys/dev/irdma/icrdma.c
@@ -53,7 +53,7 @@
/**
* Driver version
*/
-char irdma_driver_version[] = "0.0.51-k";
+char irdma_driver_version[] = "1.1.5-k";
#define pf_if_d(peer) peer->ifp->if_dunit
@@ -103,6 +103,11 @@
(rf->protocol_used == IRDMA_IWARP_PROTOCOL_ONLY) ? "iWARP" : "RoCEv2",
rf->tun_info.roce_ena);
+ snprintf(rf->tun_info.drv_ver, IRDMA_VER_LEN, "%s", irdma_driver_version);
+ SYSCTL_ADD_STRING(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list,
+ OID_AUTO, "drv_ver", CTLFLAG_RDTUN, rf->tun_info.drv_ver,
+ IRDMA_VER_LEN, "driver version");
+
irdma_dcqcn_tunables_init(rf);
}
diff --git a/sys/dev/irdma/icrdma_hw.h b/sys/dev/irdma/icrdma_hw.h
--- a/sys/dev/irdma/icrdma_hw.h
+++ b/sys/dev/irdma/icrdma_hw.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2019 - 2020 Intel Corporation
+ * Copyright (c) 2017 - 2022 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -101,28 +101,29 @@
#define ICRDMA_VF_DB_ADDR_OFFSET (64 * 1024)
-/* CCQSTATUS */
-#define ICRDMA_CCQPSTATUS_CCQP_DONE_S 0
-#define ICRDMA_CCQPSTATUS_CCQP_DONE_M (0x1ULL << ICRDMA_CCQPSTATUS_CCQP_DONE_S)
-#define ICRDMA_CCQPSTATUS_CCQP_ERR_S 31
-#define ICRDMA_CCQPSTATUS_CCQP_ERR_M (0x1ULL << ICRDMA_CCQPSTATUS_CCQP_ERR_S)
-#define ICRDMA_CQPSQ_STAG_PDID_S 46
-#define ICRDMA_CQPSQ_STAG_PDID_M (0x3ffffULL << ICRDMA_CQPSQ_STAG_PDID_S)
-#define ICRDMA_CQPSQ_CQ_CEQID_S 22
-#define ICRDMA_CQPSQ_CQ_CEQID_M (0x3ffULL << ICRDMA_CQPSQ_CQ_CEQID_S)
-#define ICRDMA_CQPSQ_CQ_CQID_S 0
-#define ICRDMA_CQPSQ_CQ_CQID_M (0x7ffffULL << ICRDMA_CQPSQ_CQ_CQID_S)
-#define ICRDMA_COMMIT_FPM_CQCNT_S 0
-#define ICRDMA_COMMIT_FPM_CQCNT_M (0xfffffULL << ICRDMA_COMMIT_FPM_CQCNT_S)
+#define ICRDMA_CCQPSTATUS_CCQP_DONE_S 0
+#define ICRDMA_CCQPSTATUS_CCQP_DONE BIT_ULL(0)
+#define ICRDMA_CCQPSTATUS_CCQP_ERR_S 31
+#define ICRDMA_CCQPSTATUS_CCQP_ERR BIT_ULL(31)
+#define ICRDMA_CQPSQ_STAG_PDID_S 46
+#define ICRDMA_CQPSQ_STAG_PDID GENMASK_ULL(63, 46)
+#define ICRDMA_CQPSQ_CQ_CEQID_S 22
+#define ICRDMA_CQPSQ_CQ_CEQID GENMASK_ULL(31, 22)
+#define ICRDMA_CQPSQ_CQ_CQID_S 0
+#define ICRDMA_CQPSQ_CQ_CQID GENMASK_ULL(18, 0)
+#define ICRDMA_COMMIT_FPM_CQCNT_S 0
+#define ICRDMA_COMMIT_FPM_CQCNT GENMASK_ULL(19, 0)
+#define ICRDMA_CQPSQ_UPESD_HMCFNID_S 0
+#define ICRDMA_CQPSQ_UPESD_HMCFNID GENMASK_ULL(5, 0)
enum icrdma_device_caps_const {
ICRDMA_MAX_WQ_FRAGMENT_COUNT = 13,
ICRDMA_MAX_SGE_RD = 13,
-
ICRDMA_MAX_STATS_COUNT = 128,
- ICRDMA_MAX_IRD_SIZE = 127,
- ICRDMA_MAX_ORD_SIZE = 255,
+ ICRDMA_MAX_IRD_SIZE = 32,
+ ICRDMA_MAX_ORD_SIZE = 64,
+ ICRDMA_MIN_WQ_SIZE = 8 /* WQEs */,
};
diff --git a/sys/dev/irdma/icrdma_hw.c b/sys/dev/irdma/icrdma_hw.c
--- a/sys/dev/irdma/icrdma_hw.c
+++ b/sys/dev/irdma/icrdma_hw.c
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2017 - 2021 Intel Corporation
+ * Copyright (c) 2017 - 2022 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -71,21 +71,23 @@
};
static u64 icrdma_masks[IRDMA_MAX_MASKS] = {
- ICRDMA_CCQPSTATUS_CCQP_DONE_M,
- ICRDMA_CCQPSTATUS_CCQP_ERR_M,
- ICRDMA_CQPSQ_STAG_PDID_M,
- ICRDMA_CQPSQ_CQ_CEQID_M,
- ICRDMA_CQPSQ_CQ_CQID_M,
- ICRDMA_COMMIT_FPM_CQCNT_M,
+ ICRDMA_CCQPSTATUS_CCQP_DONE,
+ ICRDMA_CCQPSTATUS_CCQP_ERR,
+ ICRDMA_CQPSQ_STAG_PDID,
+ ICRDMA_CQPSQ_CQ_CEQID,
+ ICRDMA_CQPSQ_CQ_CQID,
+ ICRDMA_COMMIT_FPM_CQCNT,
+ ICRDMA_CQPSQ_UPESD_HMCFNID
};
-static u64 icrdma_shifts[IRDMA_MAX_SHIFTS] = {
+static u8 icrdma_shifts[IRDMA_MAX_SHIFTS] = {
ICRDMA_CCQPSTATUS_CCQP_DONE_S,
ICRDMA_CCQPSTATUS_CCQP_ERR_S,
ICRDMA_CQPSQ_STAG_PDID_S,
ICRDMA_CQPSQ_CQ_CEQID_S,
ICRDMA_CQPSQ_CQ_CQID_S,
ICRDMA_COMMIT_FPM_CQCNT_S,
+ ICRDMA_CQPSQ_UPESD_HMCFNID_S
};
/**
@@ -101,9 +103,10 @@
if (dev->ceq_itr && dev->aeq->msix_idx != idx)
interval = dev->ceq_itr >> 1; /* 2 usec units */
- val = LS_64(0, IRDMA_GLINT_DYN_CTL_ITR_INDX) |
- LS_64(interval, IRDMA_GLINT_DYN_CTL_INTERVAL) |
- IRDMA_GLINT_DYN_CTL_INTENA_M | IRDMA_GLINT_DYN_CTL_CLEARPBA_M;
+ val = FIELD_PREP(IRDMA_GLINT_DYN_CTL_ITR_INDX, IRDMA_IDX_ITR0) |
+ FIELD_PREP(IRDMA_GLINT_DYN_CTL_INTERVAL, interval) |
+ FIELD_PREP(IRDMA_GLINT_DYN_CTL_INTENA, true) |
+ FIELD_PREP(IRDMA_GLINT_DYN_CTL_CLEARPBA, true);
writel(val, dev->hw_regs[IRDMA_GLINT_DYN_CTL] + idx);
}
@@ -131,9 +134,9 @@
{
u32 reg_val;
- reg_val = enable ? IRDMA_GLINT_CEQCTL_CAUSE_ENA_M : 0;
+ reg_val = enable ? IRDMA_GLINT_CEQCTL_CAUSE_ENA : 0;
reg_val |= (idx << IRDMA_GLINT_CEQCTL_MSIX_INDX_S) |
- IRDMA_GLINT_CEQCTL_ITR_INDX_M;
+ IRDMA_GLINT_CEQCTL_ITR_INDX;
writel(reg_val, dev->hw_regs[IRDMA_GLINT_CEQCTL] + ceq_id);
}
@@ -224,7 +227,7 @@
dev->cq_ack_db = dev->hw_regs[IRDMA_CQACK];
dev->irq_ops = &icrdma_irq_ops;
dev->hw_stats_map = icrdma_hw_stat_map;
-
+ dev->hw_attrs.page_size_cap = SZ_4K | SZ_2M | SZ_1G;
dev->hw_attrs.max_hw_ird = ICRDMA_MAX_IRD_SIZE;
dev->hw_attrs.max_hw_ord = ICRDMA_MAX_ORD_SIZE;
dev->hw_attrs.max_stat_inst = ICRDMA_MAX_STATS_COUNT;
@@ -232,8 +235,7 @@
dev->hw_attrs.uk_attrs.max_hw_wq_frags = ICRDMA_MAX_WQ_FRAGMENT_COUNT;
dev->hw_attrs.uk_attrs.max_hw_read_sges = ICRDMA_MAX_SGE_RD;
- dev->hw_attrs.uk_attrs.max_hw_wq_size = IRDMA_QP_WQE_MAX_SIZE;
- dev->hw_attrs.uk_attrs.min_sw_wq_size = IRDMA_QP_SW_MIN_WQSIZE;
+ dev->hw_attrs.uk_attrs.min_hw_wq_size = ICRDMA_MIN_WQ_SIZE;
dev->hw_attrs.uk_attrs.max_hw_sq_chunk = IRDMA_MAX_QUANTA_PER_WR;
disable_tx_spad(dev->hw);
disable_prefetch(dev->hw);
@@ -320,6 +322,9 @@
#define IRDMA_CWND_NO_FC 0x1
#define IRDMA_CWND_FC 0x18
+#define IRDMA_RTOMIN_NO_FC 0x5
+#define IRDMA_RTOMIN_FC 0x32
+
#define IRDMA_ACKCREDS_NO_FC 0x02
#define IRDMA_ACKCREDS_FC 0x06
@@ -405,7 +410,7 @@
wr32(hw, GLPE_WQMTXIDXDATA, wqm_data);
}
-#define GL_RDPU_CNTRL 0x52054
+#define GL_RDPU_CNTRL 0x52054
void
rdpu_ackreqpmthresh(struct irdma_hw *hw)
{
diff --git a/sys/dev/irdma/irdma-abi.h b/sys/dev/irdma/irdma-abi.h
--- a/sys/dev/irdma/irdma-abi.h
+++ b/sys/dev/irdma/irdma-abi.h
@@ -2,7 +2,7 @@
* SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) OR Linux-OpenIB)
*
*
- * Copyright (c) 2006 - 2021 Intel Corporation. All rights reserved.
+ * Copyright (c) 2006 - 2022 Intel Corporation. All rights reserved.
* Copyright (c) 2005 Topspin Communications. All rights reserved.
* Copyright (c) 2005 Cisco Systems. All rights reserved.
* Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
diff --git a/sys/dev/irdma/irdma.h b/sys/dev/irdma/irdma.h
--- a/sys/dev/irdma/irdma.h
+++ b/sys/dev/irdma/irdma.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2017 - 2021 Intel Corporation
+ * Copyright (c) 2017 - 2022 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -39,81 +39,74 @@
#define RDMA_BIT2(type, a) ((u##type) 1UL << a)
#define RDMA_MASK3(type, mask, shift) ((u##type) mask << shift)
#define MAKEMASK(m, s) ((m) << (s))
-#define IRDMA_WQEALLOC_WQE_DESC_INDEX_S 20
-#define IRDMA_WQEALLOC_WQE_DESC_INDEX_M (0xfff << IRDMA_WQEALLOC_WQE_DESC_INDEX_S)
-#define IRDMA_CQPTAIL_WQTAIL_S 0
-#define IRDMA_CQPTAIL_WQTAIL_M (0x7ff << IRDMA_CQPTAIL_WQTAIL_S)
-
-#define IRDMA_CQPTAIL_CQP_OP_ERR_S 31
-#define IRDMA_CQPTAIL_CQP_OP_ERR_M (0x1 << IRDMA_CQPTAIL_CQP_OP_ERR_S)
-
-#define IRDMA_CQPERRCODES_CQP_MINOR_CODE_S 0
-#define IRDMA_CQPERRCODES_CQP_MINOR_CODE_M (0xffff << IRDMA_CQPERRCODES_CQP_MINOR_CODE_S)
-#define IRDMA_CQPERRCODES_CQP_MAJOR_CODE_S 16
-#define IRDMA_CQPERRCODES_CQP_MAJOR_CODE_M (0xffff << IRDMA_CQPERRCODES_CQP_MAJOR_CODE_S)
-
-#define IRDMA_GLPCI_LBARCTRL_PE_DB_SIZE_S 4
-#define IRDMA_GLPCI_LBARCTRL_PE_DB_SIZE_M (0x3 << IRDMA_GLPCI_LBARCTRL_PE_DB_SIZE_S)
-
-#define IRDMA_GLINT_RATE_INTERVAL_S 0
-#define IRDMA_GLINT_RATE_INTERVAL_M (0x3c << IRDMA_GLINT_RATE_INTERVAL_S)
-
-#define IRDMA_GLINT_RATE_INTRL_ENA_S 6
-#define IRDMA_GLINT_RATE_INTRL_ENA_M BIT(6)
-
-#define IRDMA_GLINT_DYN_CTL_INTENA_S 0
-#define IRDMA_GLINT_DYN_CTL_INTENA_M (0x1 << IRDMA_GLINT_DYN_CTL_INTENA_S)
-
-#define IRDMA_GLINT_DYN_CTL_CLEARPBA_S 1
-#define IRDMA_GLINT_DYN_CTL_CLEARPBA_M (0x1 << IRDMA_GLINT_DYN_CTL_CLEARPBA_S)
-
-#define IRDMA_GLINT_DYN_CTL_ITR_INDX_S 3
-#define IRDMA_GLINT_DYN_CTL_ITR_INDX_M (0x3 << IRDMA_GLINT_DYN_CTL_ITR_INDX_S)
-
-#define IRDMA_GLINT_DYN_CTL_INTERVAL_S 5
-#define IRDMA_GLINT_DYN_CTL_INTERVAL_M (0xfff << IRDMA_GLINT_DYN_CTL_INTERVAL_S)
-
-#define IRDMA_GLINT_CEQCTL_ITR_INDX_S 11
-#define IRDMA_GLINT_CEQCTL_ITR_INDX_M (0x3 << IRDMA_GLINT_CEQCTL_ITR_INDX_S)
-
-#define IRDMA_GLINT_CEQCTL_CAUSE_ENA_S 30
-#define IRDMA_GLINT_CEQCTL_CAUSE_ENA_M (0x1 << IRDMA_GLINT_CEQCTL_CAUSE_ENA_S)
-
-#define IRDMA_GLINT_CEQCTL_MSIX_INDX_S 0
-#define IRDMA_GLINT_CEQCTL_MSIX_INDX_M (0x7ff << IRDMA_GLINT_CEQCTL_MSIX_INDX_S)
-
-#define IRDMA_PFINT_AEQCTL_MSIX_INDX_S 0
-#define IRDMA_PFINT_AEQCTL_MSIX_INDX_M (0x7ff << IRDMA_PFINT_AEQCTL_MSIX_INDX_S)
-
-#define IRDMA_PFINT_AEQCTL_ITR_INDX_S 11
-#define IRDMA_PFINT_AEQCTL_ITR_INDX_M (0x3 << IRDMA_PFINT_AEQCTL_ITR_INDX_S)
-
-#define IRDMA_PFINT_AEQCTL_CAUSE_ENA_S 30
-#define IRDMA_PFINT_AEQCTL_CAUSE_ENA_M (0x1 << IRDMA_PFINT_AEQCTL_CAUSE_ENA_S)
-
-#define IRDMA_PFHMC_PDINV_PMSDIDX_S 0
-#define IRDMA_PFHMC_PDINV_PMSDIDX_M (0xfff << IRDMA_PFHMC_PDINV_PMSDIDX_S)
-
-#define IRDMA_PFHMC_PDINV_PMSDPARTSEL_S 15
-#define IRDMA_PFHMC_PDINV_PMSDPARTSEL_M (0x1 << IRDMA_PFHMC_PDINV_PMSDPARTSEL_S)
-
-#define IRDMA_PFHMC_PDINV_PMPDIDX_S 16
-#define IRDMA_PFHMC_PDINV_PMPDIDX_M (0x1ff << IRDMA_PFHMC_PDINV_PMPDIDX_S)
-
-#define IRDMA_PFHMC_SDDATALOW_PMSDVALID_S 0
-#define IRDMA_PFHMC_SDDATALOW_PMSDVALID_M (0x1 << IRDMA_PFHMC_SDDATALOW_PMSDVALID_S)
-#define IRDMA_PFHMC_SDDATALOW_PMSDTYPE_S 1
-#define IRDMA_PFHMC_SDDATALOW_PMSDTYPE_M (0x1 << IRDMA_PFHMC_SDDATALOW_PMSDTYPE_S)
-#define IRDMA_PFHMC_SDDATALOW_PMSDBPCOUNT_S 2
-#define IRDMA_PFHMC_SDDATALOW_PMSDBPCOUNT_M (0x3ff << IRDMA_PFHMC_SDDATALOW_PMSDBPCOUNT_S)
-#define IRDMA_PFHMC_SDDATALOW_PMSDDATALOW_S 12
-#define IRDMA_PFHMC_SDDATALOW_PMSDDATALOW_M (0xfffff << IRDMA_PFHMC_SDDATALOW_PMSDDATALOW_S)
-
-#define IRDMA_PFHMC_SDCMD_PMSDWR_S 31
-#define IRDMA_PFHMC_SDCMD_PMSDWR_M (0x1 << IRDMA_PFHMC_SDCMD_PMSDWR_S)
-
-#define IRDMA_INVALID_CQ_IDX 0xffffffff
+#define IRDMA_WQEALLOC_WQE_DESC_INDEX_S 20
+#define IRDMA_WQEALLOC_WQE_DESC_INDEX GENMASK(31, 20)
+
+#define IRDMA_CQPTAIL_WQTAIL_S 0
+#define IRDMA_CQPTAIL_WQTAIL GENMASK(10, 0)
+#define IRDMA_CQPTAIL_CQP_OP_ERR_S 31
+#define IRDMA_CQPTAIL_CQP_OP_ERR BIT(31)
+
+#define IRDMA_CQPERRCODES_CQP_MINOR_CODE_S 0
+#define IRDMA_CQPERRCODES_CQP_MINOR_CODE GENMASK(15, 0)
+#define IRDMA_CQPERRCODES_CQP_MAJOR_CODE_S 16
+#define IRDMA_CQPERRCODES_CQP_MAJOR_CODE GENMASK(31, 16)
+#define IRDMA_GLPCI_LBARCTRL_PE_DB_SIZE_S 4
+#define IRDMA_GLPCI_LBARCTRL_PE_DB_SIZE GENMASK(5, 4)
+#define IRDMA_GLINT_RATE_INTERVAL_S 0
+#define IRDMA_GLINT_RATE_INTERVAL GENMASK(4, 0)
+#define IRDMA_GLINT_RATE_INTRL_ENA_S 6
+#define IRDMA_GLINT_RATE_INTRL_ENA_M BIT(6)
+#define IRDMA_GLINT_RATE_INTRL_ENA BIT(6)
+
+#define IRDMA_GLINT_DYN_CTL_INTENA_S 0
+#define IRDMA_GLINT_DYN_CTL_INTENA BIT(0)
+#define IRDMA_GLINT_DYN_CTL_CLEARPBA_S 1
+#define IRDMA_GLINT_DYN_CTL_CLEARPBA BIT(1)
+#define IRDMA_GLINT_DYN_CTL_ITR_INDX_S 3
+#define IRDMA_GLINT_DYN_CTL_ITR_INDX GENMASK(4, 3)
+#define IRDMA_GLINT_DYN_CTL_INTERVAL_S 5
+#define IRDMA_GLINT_DYN_CTL_INTERVAL GENMASK(16, 5)
+#define IRDMA_GLINT_CEQCTL_ITR_INDX_S 11
+#define IRDMA_GLINT_CEQCTL_ITR_INDX GENMASK(12, 11)
+#define IRDMA_GLINT_CEQCTL_CAUSE_ENA_S 30
+#define IRDMA_GLINT_CEQCTL_CAUSE_ENA BIT(30)
+#define IRDMA_GLINT_CEQCTL_MSIX_INDX_S 0
+#define IRDMA_GLINT_CEQCTL_MSIX_INDX GENMASK(10, 0)
+#define IRDMA_PFINT_AEQCTL_MSIX_INDX_S 0
+#define IRDMA_PFINT_AEQCTL_MSIX_INDX GENMASK(10, 0)
+#define IRDMA_PFINT_AEQCTL_ITR_INDX_S 11
+#define IRDMA_PFINT_AEQCTL_ITR_INDX GENMASK(12, 11)
+#define IRDMA_PFINT_AEQCTL_CAUSE_ENA_S 30
+#define IRDMA_PFINT_AEQCTL_CAUSE_ENA BIT(30)
+#define IRDMA_PFHMC_PDINV_PMSDIDX_S 0
+#define IRDMA_PFHMC_PDINV_PMSDIDX GENMASK(11, 0)
+#define IRDMA_PFHMC_PDINV_PMSDPARTSEL_S 15
+#define IRDMA_PFHMC_PDINV_PMSDPARTSEL BIT(15)
+#define IRDMA_PFHMC_PDINV_PMPDIDX_S 16
+#define IRDMA_PFHMC_PDINV_PMPDIDX GENMASK(24, 16)
+#define IRDMA_PFHMC_SDDATALOW_PMSDVALID_S 0
+#define IRDMA_PFHMC_SDDATALOW_PMSDVALID BIT(0)
+#define IRDMA_PFHMC_SDDATALOW_PMSDTYPE_S 1
+#define IRDMA_PFHMC_SDDATALOW_PMSDTYPE BIT(1)
+#define IRDMA_PFHMC_SDDATALOW_PMSDBPCOUNT_S 2
+#define IRDMA_PFHMC_SDDATALOW_PMSDBPCOUNT GENMASK(11, 2)
+#define IRDMA_PFHMC_SDDATALOW_PMSDDATALOW_S 12
+#define IRDMA_PFHMC_SDDATALOW_PMSDDATALOW GENMASK(31, 12)
+#define IRDMA_PFHMC_SDCMD_PMSDWR_S 31
+#define IRDMA_PFHMC_SDCMD_PMSDWR BIT(31)
+#define IRDMA_PFHMC_SDCMD_PMSDPARTSEL_S 15
+#define IRDMA_PFHMC_SDCMD_PMSDPARTSEL BIT(15)
+
+#define IRDMA_INVALID_CQ_IDX 0xffffffff
+
+enum irdma_dyn_idx_t {
+ IRDMA_IDX_ITR0 = 0,
+ IRDMA_IDX_ITR1 = 1,
+ IRDMA_IDX_ITR2 = 2,
+ IRDMA_IDX_NOITR = 3,
+};
enum irdma_registers {
IRDMA_CQPTAIL,
@@ -149,6 +142,7 @@
IRDMA_CQPSQ_CQ_CEQID_S,
IRDMA_CQPSQ_CQ_CQID_S,
IRDMA_COMMIT_FPM_CQCNT_S,
+ IRDMA_CQPSQ_UPESD_HMCFNID_S,
IRDMA_MAX_SHIFTS,
};
@@ -159,6 +153,7 @@
IRDMA_CQPSQ_CQ_CEQID_M,
IRDMA_CQPSQ_CQ_CQID_M,
IRDMA_COMMIT_FPM_CQCNT_M,
+ IRDMA_CQPSQ_UPESD_HMCFNID_M,
IRDMA_MAX_MASKS, /* Must be last entry */
};
@@ -174,7 +169,7 @@
struct irdma_mcast_grp_info {
u8 dest_mac_addr[ETH_ALEN];
u16 vlan_id;
- u8 hmc_fcn_id;
+ u16 hmc_fcn_id;
bool ipv4_valid:1;
bool vlan_valid:1;
u16 mg_id;
@@ -186,9 +181,10 @@
};
enum irdma_vers {
- IRDMA_GEN_RSVD,
- IRDMA_GEN_1,
- IRDMA_GEN_2,
+ IRDMA_GEN_RSVD = 0,
+ IRDMA_GEN_1 = 1,
+ IRDMA_GEN_2 = 2,
+ IRDMA_GEN_MAX = 2,
};
struct irdma_uk_attrs {
@@ -201,8 +197,7 @@
u32 min_hw_cq_size;
u32 max_hw_cq_size;
u16 max_hw_sq_chunk;
- u16 max_hw_wq_size;
- u16 min_sw_wq_size;
+ u16 min_hw_wq_size;
u8 hw_rev;
};
@@ -211,6 +206,7 @@
u64 max_hw_outbound_msg_size;
u64 max_hw_inbound_msg_size;
u64 max_mr_size;
+ u64 page_size_cap;
u32 min_hw_qp_id;
u32 min_hw_aeq_size;
u32 max_hw_aeq_size;
diff --git a/sys/dev/irdma/irdma_cm.h b/sys/dev/irdma/irdma_cm.h
--- a/sys/dev/irdma/irdma_cm.h
+++ b/sys/dev/irdma/irdma_cm.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2021 Intel Corporation
+ * Copyright (c) 2015 - 2022 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -74,7 +74,7 @@
#define TCP_OPTIONS_PADDING 3
#define IRDMA_DEFAULT_RETRYS 64
-#define IRDMA_DEFAULT_RETRANS 8
+#define IRDMA_DEFAULT_RETRANS 32
#define IRDMA_DEFAULT_TTL 0x40
#define IRDMA_DEFAULT_RTT_VAR 6
#define IRDMA_DEFAULT_SS_THRESH 0x3fffffff
@@ -192,14 +192,6 @@
IRDMA_CM_EVENT_ABORTED,
};
-struct irdma_bth { /* Base Trasnport Header */
- u8 opcode;
- u8 flags;
- __be16 pkey;
- __be32 qpn;
- __be32 apsn;
-};
-
struct ietf_mpa_v1 {
u8 key[IETF_MPA_KEY_SIZE];
u8 flags;
@@ -426,8 +418,8 @@
static inline u8 irdma_tos2dscp(u8 tos)
{
#define IRDMA_DSCP_S 2
-#define IRDMA_DSCP_M (0x3f << IRDMA_DSCP_S)
- return RS_32(tos, IRDMA_DSCP);
+#define IRDMA_DSCP GENMASK(7, 2)
+ return FIELD_GET(IRDMA_DSCP, tos);
}
int irdma_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param);
@@ -435,16 +427,16 @@
int irdma_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param);
int irdma_create_listen(struct iw_cm_id *cm_id, int backlog);
int irdma_destroy_listen(struct iw_cm_id *cm_id);
-int irdma_add_arp(struct irdma_pci_f *rf, u32 *ip, u8 *mac);
+int irdma_add_arp(struct irdma_pci_f *rf, u32 *ip, const u8 *mac);
void irdma_cm_teardown_connections(struct irdma_device *iwdev, u32 *ipaddr,
struct irdma_cm_info *nfo,
bool disconnect_all);
int irdma_cm_start(struct irdma_device *dev);
int irdma_cm_stop(struct irdma_device *dev);
-bool irdma_ipv4_is_lpb(struct vnet *, u32 loc_addr, u32 rem_addr);
+bool irdma_ipv4_is_lpb(u32 loc_addr, u32 rem_addr);
bool irdma_ipv6_is_lpb(u32 *loc_addr, u32 *rem_addr);
int irdma_arp_table(struct irdma_pci_f *rf, u32 *ip_addr,
- u8 *mac_addr, u32 action);
+ const u8 *mac_addr, u32 action);
bool irdma_port_in_use(struct irdma_cm_core *cm_core, u16 port);
void irdma_send_ack(struct irdma_cm_node *cm_node);
void irdma_lpb_nop(struct irdma_sc_qp *qp);
diff --git a/sys/dev/irdma/irdma_cm.c b/sys/dev/irdma/irdma_cm.c
--- a/sys/dev/irdma/irdma_cm.c
+++ b/sys/dev/irdma/irdma_cm.c
@@ -1628,6 +1628,7 @@
{
struct ifnet *ip_dev = NULL;
struct in6_addr laddr6;
+ u16 scope_id = 0;
irdma_copy_ip_htonl(laddr6.__u6_addr.__u6_addr32, addr);
if (vlan_id)
@@ -1635,7 +1636,11 @@
if (mac)
eth_zero_addr(mac);
- ip_dev = ip6_ifp_find(&init_net, laddr6, 0);
+ if (IN6_IS_SCOPE_LINKLOCAL(&laddr6) ||
+ IN6_IS_ADDR_MC_INTFACELOCAL(&laddr6))
+ scope_id = ntohs(laddr6.__u6_addr.__u6_addr16[1]);
+
+ ip_dev = ip6_ifp_find(&init_net, laddr6, scope_id);
if (ip_dev) {
if (vlan_id)
*vlan_id = rdma_vlan_dev_vlan_id(ip_dev);
@@ -2055,15 +2060,9 @@
* @rem_addr: remote address
*/
bool
-irdma_ipv4_is_lpb(struct vnet *vnet, u32 loc_addr, u32 rem_addr)
+irdma_ipv4_is_lpb(u32 loc_addr, u32 rem_addr)
{
- bool ret;
-
- CURVNET_SET_QUIET(vnet);
- ret = ipv4_is_loopback(htonl(rem_addr)) || (loc_addr == rem_addr);
- CURVNET_RESTORE();
-
- return (ret);
+ return ipv4_is_loopback(htonl(rem_addr)) || (loc_addr == rem_addr);
}
/**
@@ -2089,10 +2088,12 @@
static int
irdma_cm_create_ah(struct irdma_cm_node *cm_node, bool wait)
{
- struct rdma_cm_id *rdma_id = (struct rdma_cm_id *)cm_node->cm_id->context;
- struct vnet *vnet = rdma_id->route.addr.dev_addr.net;
struct irdma_ah_info ah_info = {0};
struct irdma_device *iwdev = cm_node->iwdev;
+#ifdef VIMAGE
+ struct rdma_cm_id *rdma_id = (struct rdma_cm_id *)cm_node->cm_id->context;
+ struct vnet *vnet = rdma_id->route.addr.dev_addr.net;
+#endif
ether_addr_copy(ah_info.mac_addr, IF_LLADDR(iwdev->netdev));
@@ -2104,9 +2105,12 @@
ah_info.ipv4_valid = true;
ah_info.dest_ip_addr[0] = cm_node->rem_addr[0];
ah_info.src_ip_addr[0] = cm_node->loc_addr[0];
- ah_info.do_lpbk = irdma_ipv4_is_lpb(vnet,
- ah_info.src_ip_addr[0],
+#ifdef VIMAGE
+ CURVNET_SET_QUIET(vnet);
+ ah_info.do_lpbk = irdma_ipv4_is_lpb(ah_info.src_ip_addr[0],
ah_info.dest_ip_addr[0]);
+ CURVNET_RESTORE();
+#endif
} else {
memcpy(ah_info.dest_ip_addr, cm_node->rem_addr,
sizeof(ah_info.dest_ip_addr));
@@ -2235,10 +2239,8 @@
}
static void
-irdma_cm_node_free_cb(struct rcu_head *rcu_head)
+irdma_destroy_connection(struct irdma_cm_node *cm_node)
{
- struct irdma_cm_node *cm_node =
- container_of(rcu_head, struct irdma_cm_node, rcu_head);
struct irdma_cm_core *cm_core = cm_node->cm_core;
struct irdma_qp *iwqp;
struct irdma_cm_info nfo;
@@ -2286,7 +2288,6 @@
}
cm_core->cm_free_ah(cm_node);
- kfree(cm_node);
}
/**
@@ -2314,8 +2315,9 @@
spin_unlock_irqrestore(&cm_core->ht_lock, flags);
- /* wait for all list walkers to exit their grace period */
- call_rcu(&cm_node->rcu_head, irdma_cm_node_free_cb);
+ irdma_destroy_connection(cm_node);
+
+ kfree_rcu(cm_node, rcu_head);
}
/**
@@ -3410,12 +3412,6 @@
}
cm_id = iwqp->cm_id;
- /* make sure we havent already closed this connection */
- if (!cm_id) {
- spin_unlock_irqrestore(&iwqp->lock, flags);
- return;
- }
-
original_hw_tcp_state = iwqp->hw_tcp_state;
original_ibqp_state = iwqp->ibqp_state;
last_ae = iwqp->last_aeq;
@@ -3437,11 +3433,11 @@
disconn_status = -ECONNRESET;
}
- if ((original_hw_tcp_state == IRDMA_TCP_STATE_CLOSED ||
- original_hw_tcp_state == IRDMA_TCP_STATE_TIME_WAIT ||
- last_ae == IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE ||
- last_ae == IRDMA_AE_BAD_CLOSE ||
- last_ae == IRDMA_AE_LLP_CONNECTION_RESET || iwdev->rf->reset)) {
+ if (original_hw_tcp_state == IRDMA_TCP_STATE_CLOSED ||
+ original_hw_tcp_state == IRDMA_TCP_STATE_TIME_WAIT ||
+ last_ae == IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE ||
+ last_ae == IRDMA_AE_BAD_CLOSE ||
+ last_ae == IRDMA_AE_LLP_CONNECTION_RESET || iwdev->rf->reset || !cm_id) {
issue_close = 1;
iwqp->cm_id = NULL;
qp->term_flags = 0;
@@ -3453,10 +3449,6 @@
spin_unlock_irqrestore(&iwqp->lock, flags);
if (issue_flush && !iwqp->sc_qp.qp_uk.destroy_pending) {
- if (!iwqp->user_mode)
- queue_delayed_work(iwqp->iwdev->cleanup_wq,
- &iwqp->dwork_flush,
- msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS));
irdma_flush_wqes(iwqp, IRDMA_FLUSH_SQ | IRDMA_FLUSH_RQ |
IRDMA_FLUSH_WAIT);
@@ -4193,10 +4185,6 @@
struct irdma_cm_node *cm_node;
struct list_head teardown_list;
struct ib_qp_attr attr;
- struct irdma_sc_vsi *vsi = &iwdev->vsi;
- struct irdma_sc_qp *sc_qp;
- struct irdma_qp *qp;
- int i;
INIT_LIST_HEAD(&teardown_list);
@@ -4213,50 +4201,4 @@
irdma_cm_disconn(cm_node->iwqp);
irdma_rem_ref_cm_node(cm_node);
}
- if (!iwdev->roce_mode)
- return;
-
- INIT_LIST_HEAD(&teardown_list);
- for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) {
- mutex_lock(&vsi->qos[i].qos_mutex);
- list_for_each_safe(list_node, list_core_temp,
- &vsi->qos[i].qplist) {
- u32 qp_ip[4];
-
- sc_qp = container_of(list_node, struct irdma_sc_qp,
- list);
- if (sc_qp->qp_uk.qp_type != IRDMA_QP_TYPE_ROCE_RC)
- continue;
-
- qp = sc_qp->qp_uk.back_qp;
- if (!disconnect_all) {
- if (nfo->ipv4)
- qp_ip[0] = qp->udp_info.local_ipaddr[3];
- else
- memcpy(qp_ip,
- &qp->udp_info.local_ipaddr[0],
- sizeof(qp_ip));
- }
-
- if (disconnect_all ||
- (nfo->vlan_id == (qp->udp_info.vlan_tag & EVL_VLID_MASK) &&
- !memcmp(qp_ip, ipaddr, nfo->ipv4 ? 4 : 16))) {
- spin_lock(&iwdev->rf->qptable_lock);
- if (iwdev->rf->qp_table[sc_qp->qp_uk.qp_id]) {
- irdma_qp_add_ref(&qp->ibqp);
- list_add(&qp->teardown_entry,
- &teardown_list);
- }
- spin_unlock(&iwdev->rf->qptable_lock);
- }
- }
- mutex_unlock(&vsi->qos[i].qos_mutex);
- }
-
- list_for_each_safe(list_node, list_core_temp, &teardown_list) {
- qp = container_of(list_node, struct irdma_qp, teardown_entry);
- attr.qp_state = IB_QPS_ERR;
- irdma_modify_qp_roce(&qp->ibqp, &attr, IB_QP_STATE, NULL);
- irdma_qp_rem_ref(&qp->ibqp);
- }
}
diff --git a/sys/dev/irdma/irdma_ctrl.c b/sys/dev/irdma/irdma_ctrl.c
--- a/sys/dev/irdma/irdma_ctrl.c
+++ b/sys/dev/irdma/irdma_ctrl.c
@@ -156,18 +156,16 @@
irdma_change_l2params(struct irdma_sc_vsi *vsi,
struct irdma_l2params *l2params)
{
+ if (l2params->tc_changed) {
+ vsi->tc_change_pending = false;
+ irdma_set_qos_info(vsi, l2params);
+ irdma_sc_suspend_resume_qps(vsi, IRDMA_OP_RESUME);
+ }
if (l2params->mtu_changed) {
vsi->mtu = l2params->mtu;
if (vsi->ieq)
irdma_reinitialize_ieq(vsi);
}
-
- if (!l2params->tc_changed)
- return;
-
- vsi->tc_change_pending = false;
- irdma_set_qos_info(vsi, l2params);
- irdma_sc_suspend_resume_qps(vsi, IRDMA_OP_RESUME);
}
/**
@@ -253,10 +251,10 @@
set_64bit_val(wqe, IRDMA_BYTE_16, temp);
hdr = info->arp_index |
- LS_64(IRDMA_CQP_OP_MANAGE_ARP, IRDMA_CQPSQ_OPCODE) |
- LS_64((info->permanent ? 1 : 0), IRDMA_CQPSQ_MAT_PERMANENT) |
- LS_64(1, IRDMA_CQPSQ_MAT_ENTRYVALID) |
- LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MANAGE_ARP) |
+ FIELD_PREP(IRDMA_CQPSQ_MAT_PERMANENT, info->permanent) |
+ FIELD_PREP(IRDMA_CQPSQ_MAT_ENTRYVALID, true) |
+ FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
@@ -287,8 +285,9 @@
if (!wqe)
return -ENOSPC;
- hdr = arp_index | LS_64(IRDMA_CQP_OP_MANAGE_ARP, IRDMA_CQPSQ_OPCODE) |
- LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ hdr = arp_index |
+ FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MANAGE_ARP) |
+ FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
@@ -322,9 +321,9 @@
set_64bit_val(wqe, IRDMA_BYTE_16, info->port);
- hdr = LS_64(IRDMA_CQP_OP_MANAGE_APBVT, IRDMA_CQPSQ_OPCODE) |
- LS_64(info->add, IRDMA_CQPSQ_MAPT_ADDPORT) |
- LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MANAGE_APBVT) |
+ FIELD_PREP(IRDMA_CQPSQ_MAPT_ADDPORT, info->add) |
+ FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
@@ -375,48 +374,48 @@
LS_64_1(info->mac_addr[1], 32) | LS_64_1(info->mac_addr[0], 40);
set_64bit_val(wqe, IRDMA_BYTE_0, temp);
- qw1 = LS_64(info->qp_num, IRDMA_CQPSQ_QHASH_QPN) |
- LS_64(info->dest_port, IRDMA_CQPSQ_QHASH_DEST_PORT);
+ qw1 = FIELD_PREP(IRDMA_CQPSQ_QHASH_QPN, info->qp_num) |
+ FIELD_PREP(IRDMA_CQPSQ_QHASH_DEST_PORT, info->dest_port);
if (info->ipv4_valid) {
set_64bit_val(wqe, IRDMA_BYTE_48,
- LS_64(info->dest_ip[0], IRDMA_CQPSQ_QHASH_ADDR3));
+ FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info->dest_ip[0]));
} else {
set_64bit_val(wqe, IRDMA_BYTE_56,
- LS_64(info->dest_ip[0], IRDMA_CQPSQ_QHASH_ADDR0) |
- LS_64(info->dest_ip[1], IRDMA_CQPSQ_QHASH_ADDR1));
+ FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR0, info->dest_ip[0]) |
+ FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR1, info->dest_ip[1]));
set_64bit_val(wqe, IRDMA_BYTE_48,
- LS_64(info->dest_ip[2], IRDMA_CQPSQ_QHASH_ADDR2) |
- LS_64(info->dest_ip[3], IRDMA_CQPSQ_QHASH_ADDR3));
+ FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR2, info->dest_ip[2]) |
+ FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info->dest_ip[3]));
}
- qw2 = LS_64(vsi->qos[info->user_pri].qs_handle,
- IRDMA_CQPSQ_QHASH_QS_HANDLE);
+ qw2 = FIELD_PREP(IRDMA_CQPSQ_QHASH_QS_HANDLE,
+ vsi->qos[info->user_pri].qs_handle);
if (info->vlan_valid)
- qw2 |= LS_64(info->vlan_id, IRDMA_CQPSQ_QHASH_VLANID);
+ qw2 |= FIELD_PREP(IRDMA_CQPSQ_QHASH_VLANID, info->vlan_id);
set_64bit_val(wqe, IRDMA_BYTE_16, qw2);
if (info->entry_type == IRDMA_QHASH_TYPE_TCP_ESTABLISHED) {
- qw1 |= LS_64(info->src_port, IRDMA_CQPSQ_QHASH_SRC_PORT);
+ qw1 |= FIELD_PREP(IRDMA_CQPSQ_QHASH_SRC_PORT, info->src_port);
if (!info->ipv4_valid) {
set_64bit_val(wqe, IRDMA_BYTE_40,
- LS_64(info->src_ip[0], IRDMA_CQPSQ_QHASH_ADDR0) |
- LS_64(info->src_ip[1], IRDMA_CQPSQ_QHASH_ADDR1));
+ FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR0, info->src_ip[0]) |
+ FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR1, info->src_ip[1]));
set_64bit_val(wqe, IRDMA_BYTE_32,
- LS_64(info->src_ip[2], IRDMA_CQPSQ_QHASH_ADDR2) |
- LS_64(info->src_ip[3], IRDMA_CQPSQ_QHASH_ADDR3));
+ FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR2, info->src_ip[2]) |
+ FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info->src_ip[3]));
} else {
set_64bit_val(wqe, IRDMA_BYTE_32,
- LS_64(info->src_ip[0], IRDMA_CQPSQ_QHASH_ADDR3));
+ FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info->src_ip[0]));
}
}
set_64bit_val(wqe, IRDMA_BYTE_8, qw1);
- temp = LS_64(cqp->polarity, IRDMA_CQPSQ_QHASH_WQEVALID) |
- LS_64(IRDMA_CQP_OP_MANAGE_QUAD_HASH_TABLE_ENTRY,
- IRDMA_CQPSQ_QHASH_OPCODE) |
- LS_64(info->manage, IRDMA_CQPSQ_QHASH_MANAGE) |
- LS_64(info->ipv4_valid, IRDMA_CQPSQ_QHASH_IPV4VALID) |
- LS_64(info->vlan_valid, IRDMA_CQPSQ_QHASH_VLANVALID) |
- LS_64(info->entry_type, IRDMA_CQPSQ_QHASH_ENTRYTYPE);
+ temp = FIELD_PREP(IRDMA_CQPSQ_QHASH_WQEVALID, cqp->polarity) |
+ FIELD_PREP(IRDMA_CQPSQ_QHASH_OPCODE,
+ IRDMA_CQP_OP_MANAGE_QUAD_HASH_TABLE_ENTRY) |
+ FIELD_PREP(IRDMA_CQPSQ_QHASH_MANAGE, info->manage) |
+ FIELD_PREP(IRDMA_CQPSQ_QHASH_IPV4VALID, info->ipv4_valid) |
+ FIELD_PREP(IRDMA_CQPSQ_QHASH_VLANVALID, info->vlan_valid) |
+ FIELD_PREP(IRDMA_CQPSQ_QHASH_ENTRYTYPE, info->entry_type);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, temp);
@@ -471,7 +470,6 @@
return -EINVAL;
qp->llp_stream_handle = (void *)(-1);
- qp->qp_uk.force_fence = true;
qp->hw_sq_size = irdma_get_encoded_wqe_size(qp->qp_uk.sq_ring.size,
IRDMA_QUEUE_TYPE_SQ_RQ);
irdma_debug(qp->dev, IRDMA_DEBUG_WQE,
@@ -533,17 +531,19 @@
set_64bit_val(wqe, IRDMA_BYTE_40, qp->shadow_area_pa);
hdr = qp->qp_uk.qp_id |
- LS_64(IRDMA_CQP_OP_CREATE_QP, IRDMA_CQPSQ_OPCODE) |
- LS_64((info->ord_valid ? 1 : 0), IRDMA_CQPSQ_QP_ORDVALID) |
- LS_64(info->tcp_ctx_valid, IRDMA_CQPSQ_QP_TOECTXVALID) |
- LS_64(info->mac_valid, IRDMA_CQPSQ_QP_MACVALID) |
- LS_64(qp->qp_uk.qp_type, IRDMA_CQPSQ_QP_QPTYPE) |
- LS_64(qp->virtual_map, IRDMA_CQPSQ_QP_VQ) |
- LS_64(info->force_lpb, IRDMA_CQPSQ_QP_FORCELOOPBACK) |
- LS_64(info->cq_num_valid, IRDMA_CQPSQ_QP_CQNUMVALID) |
- LS_64(info->arp_cache_idx_valid, IRDMA_CQPSQ_QP_ARPTABIDXVALID) |
- LS_64(info->next_iwarp_state, IRDMA_CQPSQ_QP_NEXTIWSTATE) |
- LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_QP) |
+ FIELD_PREP(IRDMA_CQPSQ_QP_ORDVALID, info->ord_valid) |
+ FIELD_PREP(IRDMA_CQPSQ_QP_TOECTXVALID, info->tcp_ctx_valid) |
+ FIELD_PREP(IRDMA_CQPSQ_QP_MACVALID, info->mac_valid) |
+ FIELD_PREP(IRDMA_CQPSQ_QP_QPTYPE, qp->qp_uk.qp_type) |
+ FIELD_PREP(IRDMA_CQPSQ_QP_VQ, qp->virtual_map) |
+ FIELD_PREP(IRDMA_CQPSQ_QP_FORCELOOPBACK, info->force_lpb) |
+ FIELD_PREP(IRDMA_CQPSQ_QP_CQNUMVALID, info->cq_num_valid) |
+ FIELD_PREP(IRDMA_CQPSQ_QP_ARPTABIDXVALID,
+ info->arp_cache_idx_valid) |
+ FIELD_PREP(IRDMA_CQPSQ_QP_NEXTIWSTATE, info->next_iwarp_state) |
+ FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
+
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
@@ -589,28 +589,32 @@
}
set_64bit_val(wqe, IRDMA_BYTE_8,
- LS_64(info->new_mss, IRDMA_CQPSQ_QP_NEWMSS) |
- LS_64(term_len, IRDMA_CQPSQ_QP_TERMLEN));
+ FIELD_PREP(IRDMA_CQPSQ_QP_NEWMSS, info->new_mss) |
+ FIELD_PREP(IRDMA_CQPSQ_QP_TERMLEN, term_len));
set_64bit_val(wqe, IRDMA_BYTE_16, qp->hw_host_ctx_pa);
set_64bit_val(wqe, IRDMA_BYTE_40, qp->shadow_area_pa);
hdr = qp->qp_uk.qp_id |
- LS_64(IRDMA_CQP_OP_MODIFY_QP, IRDMA_CQPSQ_OPCODE) |
- LS_64(info->ord_valid, IRDMA_CQPSQ_QP_ORDVALID) |
- LS_64(info->tcp_ctx_valid, IRDMA_CQPSQ_QP_TOECTXVALID) |
- LS_64(info->cached_var_valid, IRDMA_CQPSQ_QP_CACHEDVARVALID) |
- LS_64(qp->virtual_map, IRDMA_CQPSQ_QP_VQ) |
- LS_64(info->force_lpb, IRDMA_CQPSQ_QP_FORCELOOPBACK) |
- LS_64(info->cq_num_valid, IRDMA_CQPSQ_QP_CQNUMVALID) |
- LS_64(info->mac_valid, IRDMA_CQPSQ_QP_MACVALID) |
- LS_64(qp->qp_uk.qp_type, IRDMA_CQPSQ_QP_QPTYPE) |
- LS_64(info->mss_change, IRDMA_CQPSQ_QP_MSSCHANGE) |
- LS_64(info->remove_hash_idx, IRDMA_CQPSQ_QP_REMOVEHASHENTRY) |
- LS_64(term_actions, IRDMA_CQPSQ_QP_TERMACT) |
- LS_64(info->reset_tcp_conn, IRDMA_CQPSQ_QP_RESETCON) |
- LS_64(info->arp_cache_idx_valid, IRDMA_CQPSQ_QP_ARPTABIDXVALID) |
- LS_64(info->next_iwarp_state, IRDMA_CQPSQ_QP_NEXTIWSTATE) |
- LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MODIFY_QP) |
+ FIELD_PREP(IRDMA_CQPSQ_QP_ORDVALID, info->ord_valid) |
+ FIELD_PREP(IRDMA_CQPSQ_QP_TOECTXVALID, info->tcp_ctx_valid) |
+ FIELD_PREP(IRDMA_CQPSQ_QP_CACHEDVARVALID,
+ info->cached_var_valid) |
+ FIELD_PREP(IRDMA_CQPSQ_QP_VQ, qp->virtual_map) |
+ FIELD_PREP(IRDMA_CQPSQ_QP_FORCELOOPBACK, info->force_lpb) |
+ FIELD_PREP(IRDMA_CQPSQ_QP_CQNUMVALID, info->cq_num_valid) |
+ FIELD_PREP(IRDMA_CQPSQ_QP_MACVALID, info->mac_valid) |
+ FIELD_PREP(IRDMA_CQPSQ_QP_QPTYPE, qp->qp_uk.qp_type) |
+ FIELD_PREP(IRDMA_CQPSQ_QP_MSSCHANGE, info->mss_change) |
+ FIELD_PREP(IRDMA_CQPSQ_QP_REMOVEHASHENTRY,
+ info->remove_hash_idx) |
+ FIELD_PREP(IRDMA_CQPSQ_QP_TERMACT, term_actions) |
+ FIELD_PREP(IRDMA_CQPSQ_QP_RESETCON, info->reset_tcp_conn) |
+ FIELD_PREP(IRDMA_CQPSQ_QP_ARPTABIDXVALID,
+ info->arp_cache_idx_valid) |
+ FIELD_PREP(IRDMA_CQPSQ_QP_NEXTIWSTATE, info->next_iwarp_state) |
+ FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
+
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
@@ -648,11 +652,11 @@
set_64bit_val(wqe, IRDMA_BYTE_40, qp->shadow_area_pa);
hdr = qp->qp_uk.qp_id |
- LS_64(IRDMA_CQP_OP_DESTROY_QP, IRDMA_CQPSQ_OPCODE) |
- LS_64(qp->qp_uk.qp_type, IRDMA_CQPSQ_QP_QPTYPE) |
- LS_64(ignore_mw_bnd, IRDMA_CQPSQ_QP_IGNOREMWBOUND) |
- LS_64(remove_hash_idx, IRDMA_CQPSQ_QP_REMOVEHASHENTRY) |
- LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_QP) |
+ FIELD_PREP(IRDMA_CQPSQ_QP_QPTYPE, qp->qp_uk.qp_type) |
+ FIELD_PREP(IRDMA_CQPSQ_QP_IGNOREMWBOUND, ignore_mw_bnd) |
+ FIELD_PREP(IRDMA_CQPSQ_QP_REMOVEHASHENTRY, remove_hash_idx) |
+ FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
@@ -728,20 +732,20 @@
push_idx = qp->push_idx;
}
set_64bit_val(qp_ctx, IRDMA_BYTE_0,
- LS_64(qp->qp_uk.rq_wqe_size, IRDMAQPC_RQWQESIZE) |
- LS_64(qp->rcv_tph_en, IRDMAQPC_RCVTPHEN) |
- LS_64(qp->xmit_tph_en, IRDMAQPC_XMITTPHEN) |
- LS_64(qp->rq_tph_en, IRDMAQPC_RQTPHEN) |
- LS_64(qp->sq_tph_en, IRDMAQPC_SQTPHEN) |
- LS_64(push_idx, IRDMAQPC_PPIDX) |
- LS_64(push_mode_en, IRDMAQPC_PMENA) |
- LS_64(roce_info->pd_id >> 16, IRDMAQPC_PDIDXHI) |
- LS_64(roce_info->dctcp_en, IRDMAQPC_DC_TCP_EN) |
- LS_64(roce_info->err_rq_idx_valid, IRDMAQPC_ERR_RQ_IDX_VALID) |
- LS_64(roce_info->is_qp1, IRDMAQPC_ISQP1) |
- LS_64(roce_info->roce_tver, IRDMAQPC_ROCE_TVER) |
- LS_64(udp->ipv4, IRDMAQPC_IPV4) |
- LS_64(udp->insert_vlan_tag, IRDMAQPC_INSERTVLANTAG));
+ FIELD_PREP(IRDMAQPC_RQWQESIZE, qp->qp_uk.rq_wqe_size) |
+ FIELD_PREP(IRDMAQPC_RCVTPHEN, qp->rcv_tph_en) |
+ FIELD_PREP(IRDMAQPC_XMITTPHEN, qp->xmit_tph_en) |
+ FIELD_PREP(IRDMAQPC_RQTPHEN, qp->rq_tph_en) |
+ FIELD_PREP(IRDMAQPC_SQTPHEN, qp->sq_tph_en) |
+ FIELD_PREP(IRDMAQPC_PPIDX, push_idx) |
+ FIELD_PREP(IRDMAQPC_PMENA, push_mode_en) |
+ FIELD_PREP(IRDMAQPC_PDIDXHI, roce_info->pd_id >> 16) |
+ FIELD_PREP(IRDMAQPC_DC_TCP_EN, roce_info->dctcp_en) |
+ FIELD_PREP(IRDMAQPC_ERR_RQ_IDX_VALID, roce_info->err_rq_idx_valid) |
+ FIELD_PREP(IRDMAQPC_ISQP1, roce_info->is_qp1) |
+ FIELD_PREP(IRDMAQPC_ROCE_TVER, roce_info->roce_tver) |
+ FIELD_PREP(IRDMAQPC_IPV4, udp->ipv4) |
+ FIELD_PREP(IRDMAQPC_INSERTVLANTAG, udp->insert_vlan_tag));
set_64bit_val(qp_ctx, IRDMA_BYTE_8, qp->sq_pa);
set_64bit_val(qp_ctx, IRDMA_BYTE_16, qp->rq_pa);
if (roce_info->dcqcn_en || roce_info->dctcp_en) {
@@ -750,80 +754,81 @@
}
set_64bit_val(qp_ctx, IRDMA_BYTE_24,
- LS_64(qp->hw_rq_size, IRDMAQPC_RQSIZE) |
- LS_64(qp->hw_sq_size, IRDMAQPC_SQSIZE) |
- LS_64(udp->ttl, IRDMAQPC_TTL) | LS_64(udp->tos, IRDMAQPC_TOS) |
- LS_64(udp->src_port, IRDMAQPC_SRCPORTNUM) |
- LS_64(udp->dst_port, IRDMAQPC_DESTPORTNUM));
+ FIELD_PREP(IRDMAQPC_RQSIZE, qp->hw_rq_size) |
+ FIELD_PREP(IRDMAQPC_SQSIZE, qp->hw_sq_size) |
+ FIELD_PREP(IRDMAQPC_TTL, udp->ttl) | FIELD_PREP(IRDMAQPC_TOS, udp->tos) |
+ FIELD_PREP(IRDMAQPC_SRCPORTNUM, udp->src_port) |
+ FIELD_PREP(IRDMAQPC_DESTPORTNUM, udp->dst_port));
set_64bit_val(qp_ctx, IRDMA_BYTE_32,
- LS_64(udp->dest_ip_addr[2], IRDMAQPC_DESTIPADDR2) |
- LS_64(udp->dest_ip_addr[3], IRDMAQPC_DESTIPADDR3));
+ FIELD_PREP(IRDMAQPC_DESTIPADDR2, udp->dest_ip_addr[2]) |
+ FIELD_PREP(IRDMAQPC_DESTIPADDR3, udp->dest_ip_addr[3]));
set_64bit_val(qp_ctx, IRDMA_BYTE_40,
- LS_64(udp->dest_ip_addr[0], IRDMAQPC_DESTIPADDR0) |
- LS_64(udp->dest_ip_addr[1], IRDMAQPC_DESTIPADDR1));
+ FIELD_PREP(IRDMAQPC_DESTIPADDR0, udp->dest_ip_addr[0]) |
+ FIELD_PREP(IRDMAQPC_DESTIPADDR1, udp->dest_ip_addr[1]));
set_64bit_val(qp_ctx, IRDMA_BYTE_48,
- LS_64(udp->snd_mss, IRDMAQPC_SNDMSS) |
- LS_64(udp->vlan_tag, IRDMAQPC_VLANTAG) |
- LS_64(udp->arp_idx, IRDMAQPC_ARPIDX));
+ FIELD_PREP(IRDMAQPC_SNDMSS, udp->snd_mss) |
+ FIELD_PREP(IRDMAQPC_VLANTAG, udp->vlan_tag) |
+ FIELD_PREP(IRDMAQPC_ARPIDX, udp->arp_idx));
set_64bit_val(qp_ctx, IRDMA_BYTE_56,
- LS_64(roce_info->p_key, IRDMAQPC_PKEY) |
- LS_64(roce_info->pd_id, IRDMAQPC_PDIDX) |
- LS_64(roce_info->ack_credits, IRDMAQPC_ACKCREDITS) |
- LS_64(udp->flow_label, IRDMAQPC_FLOWLABEL));
+ FIELD_PREP(IRDMAQPC_PKEY, roce_info->p_key) |
+ FIELD_PREP(IRDMAQPC_PDIDX, roce_info->pd_id) |
+ FIELD_PREP(IRDMAQPC_ACKCREDITS, roce_info->ack_credits) |
+ FIELD_PREP(IRDMAQPC_FLOWLABEL, udp->flow_label));
set_64bit_val(qp_ctx, IRDMA_BYTE_64,
- LS_64(roce_info->qkey, IRDMAQPC_QKEY) |
- LS_64(roce_info->dest_qp, IRDMAQPC_DESTQP));
+ FIELD_PREP(IRDMAQPC_QKEY, roce_info->qkey) |
+ FIELD_PREP(IRDMAQPC_DESTQP, roce_info->dest_qp));
set_64bit_val(qp_ctx, IRDMA_BYTE_80,
- LS_64(udp->psn_nxt, IRDMAQPC_PSNNXT) |
- LS_64(udp->lsn, IRDMAQPC_LSN));
- set_64bit_val(qp_ctx, IRDMA_BYTE_88, LS_64(udp->epsn, IRDMAQPC_EPSN));
+ FIELD_PREP(IRDMAQPC_PSNNXT, udp->psn_nxt) |
+ FIELD_PREP(IRDMAQPC_LSN, udp->lsn));
+ set_64bit_val(qp_ctx, IRDMA_BYTE_88,
+ FIELD_PREP(IRDMAQPC_EPSN, udp->epsn));
set_64bit_val(qp_ctx, IRDMA_BYTE_96,
- LS_64(udp->psn_max, IRDMAQPC_PSNMAX) |
- LS_64(udp->psn_una, IRDMAQPC_PSNUNA));
+ FIELD_PREP(IRDMAQPC_PSNMAX, udp->psn_max) |
+ FIELD_PREP(IRDMAQPC_PSNUNA, udp->psn_una));
set_64bit_val(qp_ctx, IRDMA_BYTE_112,
- LS_64(udp->cwnd, IRDMAQPC_CWNDROCE));
+ FIELD_PREP(IRDMAQPC_CWNDROCE, udp->cwnd));
set_64bit_val(qp_ctx, IRDMA_BYTE_128,
- LS_64(roce_info->err_rq_idx, IRDMAQPC_ERR_RQ_IDX) |
- LS_64(udp->rnr_nak_thresh, IRDMAQPC_RNRNAK_THRESH) |
- LS_64(udp->rexmit_thresh, IRDMAQPC_REXMIT_THRESH) |
- LS_64(roce_info->rtomin, IRDMAQPC_RTOMIN));
+ FIELD_PREP(IRDMAQPC_ERR_RQ_IDX, roce_info->err_rq_idx) |
+ FIELD_PREP(IRDMAQPC_RNRNAK_THRESH, udp->rnr_nak_thresh) |
+ FIELD_PREP(IRDMAQPC_REXMIT_THRESH, udp->rexmit_thresh) |
+ FIELD_PREP(IRDMAQPC_RTOMIN, roce_info->rtomin));
set_64bit_val(qp_ctx, IRDMA_BYTE_136,
- LS_64(info->send_cq_num, IRDMAQPC_TXCQNUM) |
- LS_64(info->rcv_cq_num, IRDMAQPC_RXCQNUM));
+ FIELD_PREP(IRDMAQPC_TXCQNUM, info->send_cq_num) |
+ FIELD_PREP(IRDMAQPC_RXCQNUM, info->rcv_cq_num));
set_64bit_val(qp_ctx, IRDMA_BYTE_144,
- LS_64(info->stats_idx, IRDMAQPC_STAT_INDEX));
+ FIELD_PREP(IRDMAQPC_STAT_INDEX, info->stats_idx));
set_64bit_val(qp_ctx, IRDMA_BYTE_152, mac);
set_64bit_val(qp_ctx, IRDMA_BYTE_160,
- LS_64(roce_info->ord_size, IRDMAQPC_ORDSIZE) |
- LS_64(irdma_sc_get_encoded_ird_size(roce_info->ird_size), IRDMAQPC_IRDSIZE) |
- LS_64(roce_info->wr_rdresp_en, IRDMAQPC_WRRDRSPOK) |
- LS_64(roce_info->rd_en, IRDMAQPC_RDOK) |
- LS_64(info->stats_idx_valid, IRDMAQPC_USESTATSINSTANCE) |
- LS_64(roce_info->bind_en, IRDMAQPC_BINDEN) |
- LS_64(roce_info->fast_reg_en, IRDMAQPC_FASTREGEN) |
- LS_64(roce_info->dcqcn_en, IRDMAQPC_DCQCNENABLE) |
- LS_64(roce_info->rcv_no_icrc, IRDMAQPC_RCVNOICRC) |
- LS_64(roce_info->fw_cc_enable, IRDMAQPC_FW_CC_ENABLE) |
- LS_64(roce_info->udprivcq_en, IRDMAQPC_UDPRIVCQENABLE) |
- LS_64(roce_info->priv_mode_en, IRDMAQPC_PRIVEN) |
- LS_64(roce_info->timely_en, IRDMAQPC_TIMELYENABLE));
+ FIELD_PREP(IRDMAQPC_ORDSIZE, roce_info->ord_size) |
+ FIELD_PREP(IRDMAQPC_IRDSIZE, irdma_sc_get_encoded_ird_size(roce_info->ird_size)) |
+ FIELD_PREP(IRDMAQPC_WRRDRSPOK, roce_info->wr_rdresp_en) |
+ FIELD_PREP(IRDMAQPC_RDOK, roce_info->rd_en) |
+ FIELD_PREP(IRDMAQPC_USESTATSINSTANCE, info->stats_idx_valid) |
+ FIELD_PREP(IRDMAQPC_BINDEN, roce_info->bind_en) |
+ FIELD_PREP(IRDMAQPC_FASTREGEN, roce_info->fast_reg_en) |
+ FIELD_PREP(IRDMAQPC_DCQCNENABLE, roce_info->dcqcn_en) |
+ FIELD_PREP(IRDMAQPC_RCVNOICRC, roce_info->rcv_no_icrc) |
+ FIELD_PREP(IRDMAQPC_FW_CC_ENABLE, roce_info->fw_cc_enable) |
+ FIELD_PREP(IRDMAQPC_UDPRIVCQENABLE, roce_info->udprivcq_en) |
+ FIELD_PREP(IRDMAQPC_PRIVEN, roce_info->priv_mode_en) |
+ FIELD_PREP(IRDMAQPC_TIMELYENABLE, roce_info->timely_en));
set_64bit_val(qp_ctx, IRDMA_BYTE_168,
- LS_64(info->qp_compl_ctx, IRDMAQPC_QPCOMPCTX));
+ FIELD_PREP(IRDMAQPC_QPCOMPCTX, info->qp_compl_ctx));
set_64bit_val(qp_ctx, IRDMA_BYTE_176,
- LS_64(qp->sq_tph_val, IRDMAQPC_SQTPHVAL) |
- LS_64(qp->rq_tph_val, IRDMAQPC_RQTPHVAL) |
- LS_64(qp->qs_handle, IRDMAQPC_QSHANDLE));
+ FIELD_PREP(IRDMAQPC_SQTPHVAL, qp->sq_tph_val) |
+ FIELD_PREP(IRDMAQPC_RQTPHVAL, qp->rq_tph_val) |
+ FIELD_PREP(IRDMAQPC_QSHANDLE, qp->qs_handle));
set_64bit_val(qp_ctx, IRDMA_BYTE_184,
- LS_64(udp->local_ipaddr[3], IRDMAQPC_LOCAL_IPADDR3) |
- LS_64(udp->local_ipaddr[2], IRDMAQPC_LOCAL_IPADDR2));
+ FIELD_PREP(IRDMAQPC_LOCAL_IPADDR3, udp->local_ipaddr[3]) |
+ FIELD_PREP(IRDMAQPC_LOCAL_IPADDR2, udp->local_ipaddr[2]));
set_64bit_val(qp_ctx, IRDMA_BYTE_192,
- LS_64(udp->local_ipaddr[1], IRDMAQPC_LOCAL_IPADDR1) |
- LS_64(udp->local_ipaddr[0], IRDMAQPC_LOCAL_IPADDR0));
+ FIELD_PREP(IRDMAQPC_LOCAL_IPADDR1, udp->local_ipaddr[1]) |
+ FIELD_PREP(IRDMAQPC_LOCAL_IPADDR0, udp->local_ipaddr[0]));
set_64bit_val(qp_ctx, IRDMA_BYTE_200,
- LS_64(roce_info->t_high, IRDMAQPC_THIGH) |
- LS_64(roce_info->t_low, IRDMAQPC_TLOW));
+ FIELD_PREP(IRDMAQPC_THIGH, roce_info->t_high) |
+ FIELD_PREP(IRDMAQPC_TLOW, roce_info->t_low));
set_64bit_val(qp_ctx, IRDMA_BYTE_208,
- LS_64(info->rem_endpoint_idx, IRDMAQPC_REMENDPOINTIDX));
+ FIELD_PREP(IRDMAQPC_REMENDPOINTIDX, info->rem_endpoint_idx));
irdma_debug_buf(qp->dev, IRDMA_DEBUG_WQE, "QP_HOST CTX WQE", qp_ctx,
IRDMA_QP_CTX_SIZE);
@@ -844,9 +849,9 @@
if (!wqe)
return -ENOSPC;
- hdr = LS_64(IRDMA_CQP_OP_ALLOCATE_LOC_MAC_TABLE_ENTRY,
- IRDMA_CQPSQ_OPCODE) |
- LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE,
+ IRDMA_CQP_OP_ALLOCATE_LOC_MAC_TABLE_ENTRY) |
+ FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
@@ -884,9 +889,10 @@
set_64bit_val(wqe, IRDMA_BYTE_32, temp);
- header = LS_64(info->entry_idx, IRDMA_CQPSQ_MLM_TABLEIDX) |
- LS_64(IRDMA_CQP_OP_MANAGE_LOC_MAC_TABLE, IRDMA_CQPSQ_OPCODE) |
- LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ header = FIELD_PREP(IRDMA_CQPSQ_MLM_TABLEIDX, info->entry_idx) |
+ FIELD_PREP(IRDMA_CQPSQ_OPCODE,
+ IRDMA_CQP_OP_MANAGE_LOC_MAC_TABLE) |
+ FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
@@ -919,11 +925,12 @@
wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
if (!wqe)
return -ENOSPC;
- header = LS_64(entry_idx, IRDMA_CQPSQ_MLM_TABLEIDX) |
- LS_64(IRDMA_CQP_OP_MANAGE_LOC_MAC_TABLE, IRDMA_CQPSQ_OPCODE) |
- LS_64(1, IRDMA_CQPSQ_MLM_FREEENTRY) |
- LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID) |
- LS_64(ignore_ref_count, IRDMA_CQPSQ_MLM_IGNORE_REF_CNT);
+ header = FIELD_PREP(IRDMA_CQPSQ_MLM_TABLEIDX, entry_idx) |
+ FIELD_PREP(IRDMA_CQPSQ_OPCODE,
+ IRDMA_CQP_OP_MANAGE_LOC_MAC_TABLE) |
+ FIELD_PREP(IRDMA_CQPSQ_MLM_FREEENTRY, 1) |
+ FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity) |
+ FIELD_PREP(IRDMA_CQPSQ_MLM_IGNORE_REF_CNT, ignore_ref_count);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
@@ -970,45 +977,47 @@
push_mode_en = 1;
push_idx = qp->push_idx;
}
- qw0 = LS_64(qp->qp_uk.rq_wqe_size, IRDMAQPC_RQWQESIZE) |
- LS_64(qp->rcv_tph_en, IRDMAQPC_RCVTPHEN) |
- LS_64(qp->xmit_tph_en, IRDMAQPC_XMITTPHEN) |
- LS_64(qp->rq_tph_en, IRDMAQPC_RQTPHEN) |
- LS_64(qp->sq_tph_en, IRDMAQPC_SQTPHEN) |
- LS_64(push_idx, IRDMAQPC_PPIDX) |
- LS_64(push_mode_en, IRDMAQPC_PMENA);
+ qw0 = FIELD_PREP(IRDMAQPC_RQWQESIZE, qp->qp_uk.rq_wqe_size) |
+ FIELD_PREP(IRDMAQPC_RCVTPHEN, qp->rcv_tph_en) |
+ FIELD_PREP(IRDMAQPC_XMITTPHEN, qp->xmit_tph_en) |
+ FIELD_PREP(IRDMAQPC_RQTPHEN, qp->rq_tph_en) |
+ FIELD_PREP(IRDMAQPC_SQTPHEN, qp->sq_tph_en) |
+ FIELD_PREP(IRDMAQPC_PPIDX, push_idx) |
+ FIELD_PREP(IRDMAQPC_PMENA, push_mode_en);
set_64bit_val(qp_ctx, IRDMA_BYTE_8, qp->sq_pa);
set_64bit_val(qp_ctx, IRDMA_BYTE_16, qp->rq_pa);
- qw3 = LS_64(qp->hw_rq_size, IRDMAQPC_RQSIZE) |
- LS_64(qp->hw_sq_size, IRDMAQPC_SQSIZE);
+ qw3 = FIELD_PREP(IRDMAQPC_RQSIZE, qp->hw_rq_size) |
+ FIELD_PREP(IRDMAQPC_SQSIZE, qp->hw_sq_size);
if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
- qw3 |= LS_64(qp->src_mac_addr_idx, IRDMAQPC_GEN1_SRCMACADDRIDX);
+ qw3 |= FIELD_PREP(IRDMAQPC_GEN1_SRCMACADDRIDX,
+ qp->src_mac_addr_idx);
set_64bit_val(qp_ctx, IRDMA_BYTE_136,
- LS_64(info->send_cq_num, IRDMAQPC_TXCQNUM) |
- LS_64(info->rcv_cq_num, IRDMAQPC_RXCQNUM));
+ FIELD_PREP(IRDMAQPC_TXCQNUM, info->send_cq_num) |
+ FIELD_PREP(IRDMAQPC_RXCQNUM, info->rcv_cq_num));
set_64bit_val(qp_ctx, IRDMA_BYTE_168,
- LS_64(info->qp_compl_ctx, IRDMAQPC_QPCOMPCTX));
+ FIELD_PREP(IRDMAQPC_QPCOMPCTX, info->qp_compl_ctx));
set_64bit_val(qp_ctx, IRDMA_BYTE_176,
- LS_64(qp->sq_tph_val, IRDMAQPC_SQTPHVAL) |
- LS_64(qp->rq_tph_val, IRDMAQPC_RQTPHVAL) |
- LS_64(qp->qs_handle, IRDMAQPC_QSHANDLE) |
- LS_64(qp->ieq_qp, IRDMAQPC_EXCEPTION_LAN_QUEUE));
+ FIELD_PREP(IRDMAQPC_SQTPHVAL, qp->sq_tph_val) |
+ FIELD_PREP(IRDMAQPC_RQTPHVAL, qp->rq_tph_val) |
+ FIELD_PREP(IRDMAQPC_QSHANDLE, qp->qs_handle) |
+ FIELD_PREP(IRDMAQPC_EXCEPTION_LAN_QUEUE, qp->ieq_qp));
if (info->iwarp_info_valid) {
- qw0 |= LS_64(iw->ddp_ver, IRDMAQPC_DDP_VER) |
- LS_64(iw->rdmap_ver, IRDMAQPC_RDMAP_VER) |
- LS_64(iw->dctcp_en, IRDMAQPC_DC_TCP_EN) |
- LS_64(iw->ecn_en, IRDMAQPC_ECN_EN) |
- LS_64(iw->ib_rd_en, IRDMAQPC_IBRDENABLE) |
- LS_64(iw->pd_id >> 16, IRDMAQPC_PDIDXHI) |
- LS_64(iw->err_rq_idx_valid, IRDMAQPC_ERR_RQ_IDX_VALID);
- qw7 |= LS_64(iw->pd_id, IRDMAQPC_PDIDX);
- qw16 |= LS_64(iw->err_rq_idx, IRDMAQPC_ERR_RQ_IDX) |
- LS_64(iw->rtomin, IRDMAQPC_RTOMIN);
+ qw0 |= FIELD_PREP(IRDMAQPC_DDP_VER, iw->ddp_ver) |
+ FIELD_PREP(IRDMAQPC_RDMAP_VER, iw->rdmap_ver) |
+ FIELD_PREP(IRDMAQPC_DC_TCP_EN, iw->dctcp_en) |
+ FIELD_PREP(IRDMAQPC_ECN_EN, iw->ecn_en) |
+ FIELD_PREP(IRDMAQPC_IBRDENABLE, iw->ib_rd_en) |
+ FIELD_PREP(IRDMAQPC_PDIDXHI, iw->pd_id >> 16) |
+ FIELD_PREP(IRDMAQPC_ERR_RQ_IDX_VALID,
+ iw->err_rq_idx_valid);
+ qw7 |= FIELD_PREP(IRDMAQPC_PDIDX, iw->pd_id);
+ qw16 |= FIELD_PREP(IRDMAQPC_ERR_RQ_IDX, iw->err_rq_idx) |
+ FIELD_PREP(IRDMAQPC_RTOMIN, iw->rtomin);
set_64bit_val(qp_ctx, IRDMA_BYTE_144,
- LS_64(qp->q2_pa >> 8, IRDMAQPC_Q2ADDR) |
- LS_64(info->stats_idx, IRDMAQPC_STAT_INDEX));
+ FIELD_PREP(IRDMAQPC_Q2ADDR, qp->q2_pa >> 8) |
+ FIELD_PREP(IRDMAQPC_STAT_INDEX, info->stats_idx));
if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
mac = LS_64_1(iw->mac_addr[5], 16) |
@@ -1020,103 +1029,104 @@
}
set_64bit_val(qp_ctx, IRDMA_BYTE_152,
- mac | LS_64(iw->last_byte_sent, IRDMAQPC_LASTBYTESENT));
+ mac | FIELD_PREP(IRDMAQPC_LASTBYTESENT, iw->last_byte_sent));
set_64bit_val(qp_ctx, IRDMA_BYTE_160,
- LS_64(iw->ord_size, IRDMAQPC_ORDSIZE) |
- LS_64(irdma_sc_get_encoded_ird_size(iw->ird_size), IRDMAQPC_IRDSIZE) |
- LS_64(iw->wr_rdresp_en, IRDMAQPC_WRRDRSPOK) |
- LS_64(iw->rd_en, IRDMAQPC_RDOK) |
- LS_64(iw->snd_mark_en, IRDMAQPC_SNDMARKERS) |
- LS_64(iw->bind_en, IRDMAQPC_BINDEN) |
- LS_64(iw->fast_reg_en, IRDMAQPC_FASTREGEN) |
- LS_64(iw->priv_mode_en, IRDMAQPC_PRIVEN) |
- LS_64(info->stats_idx_valid, IRDMAQPC_USESTATSINSTANCE) |
- LS_64(1, IRDMAQPC_IWARPMODE) |
- LS_64(iw->rcv_mark_en, IRDMAQPC_RCVMARKERS) |
- LS_64(iw->align_hdrs, IRDMAQPC_ALIGNHDRS) |
- LS_64(iw->rcv_no_mpa_crc, IRDMAQPC_RCVNOMPACRC) |
- LS_64(iw->rcv_mark_offset, IRDMAQPC_RCVMARKOFFSET) |
- LS_64(iw->snd_mark_offset, IRDMAQPC_SNDMARKOFFSET) |
- LS_64(iw->timely_en, IRDMAQPC_TIMELYENABLE));
+ FIELD_PREP(IRDMAQPC_ORDSIZE, iw->ord_size) |
+ FIELD_PREP(IRDMAQPC_IRDSIZE, irdma_sc_get_encoded_ird_size(iw->ird_size)) |
+ FIELD_PREP(IRDMAQPC_WRRDRSPOK, iw->wr_rdresp_en) |
+ FIELD_PREP(IRDMAQPC_RDOK, iw->rd_en) |
+ FIELD_PREP(IRDMAQPC_SNDMARKERS, iw->snd_mark_en) |
+ FIELD_PREP(IRDMAQPC_BINDEN, iw->bind_en) |
+ FIELD_PREP(IRDMAQPC_FASTREGEN, iw->fast_reg_en) |
+ FIELD_PREP(IRDMAQPC_PRIVEN, iw->priv_mode_en) |
+ FIELD_PREP(IRDMAQPC_USESTATSINSTANCE, info->stats_idx_valid) |
+ FIELD_PREP(IRDMAQPC_IWARPMODE, 1) |
+ FIELD_PREP(IRDMAQPC_RCVMARKERS, iw->rcv_mark_en) |
+ FIELD_PREP(IRDMAQPC_ALIGNHDRS, iw->align_hdrs) |
+ FIELD_PREP(IRDMAQPC_RCVNOMPACRC, iw->rcv_no_mpa_crc) |
+ FIELD_PREP(IRDMAQPC_RCVMARKOFFSET, iw->rcv_mark_offset) |
+ FIELD_PREP(IRDMAQPC_SNDMARKOFFSET, iw->snd_mark_offset) |
+ FIELD_PREP(IRDMAQPC_TIMELYENABLE, iw->timely_en));
}
if (info->tcp_info_valid) {
- qw0 |= LS_64(tcp->ipv4, IRDMAQPC_IPV4) |
- LS_64(tcp->no_nagle, IRDMAQPC_NONAGLE) |
- LS_64(tcp->insert_vlan_tag, IRDMAQPC_INSERTVLANTAG) |
- LS_64(tcp->time_stamp, IRDMAQPC_TIMESTAMP) |
- LS_64(tcp->cwnd_inc_limit, IRDMAQPC_LIMIT) |
- LS_64(tcp->drop_ooo_seg, IRDMAQPC_DROPOOOSEG) |
- LS_64(tcp->dup_ack_thresh, IRDMAQPC_DUPACK_THRESH);
+ qw0 |= FIELD_PREP(IRDMAQPC_IPV4, tcp->ipv4) |
+ FIELD_PREP(IRDMAQPC_NONAGLE, tcp->no_nagle) |
+ FIELD_PREP(IRDMAQPC_INSERTVLANTAG,
+ tcp->insert_vlan_tag) |
+ FIELD_PREP(IRDMAQPC_TIMESTAMP, tcp->time_stamp) |
+ FIELD_PREP(IRDMAQPC_LIMIT, tcp->cwnd_inc_limit) |
+ FIELD_PREP(IRDMAQPC_DROPOOOSEG, tcp->drop_ooo_seg) |
+ FIELD_PREP(IRDMAQPC_DUPACK_THRESH, tcp->dup_ack_thresh);
if (iw->ecn_en || iw->dctcp_en) {
tcp->tos &= ~ECN_CODE_PT_MASK;
tcp->tos |= ECN_CODE_PT_VAL;
}
- qw3 |= LS_64(tcp->ttl, IRDMAQPC_TTL) |
- LS_64(tcp->avoid_stretch_ack, IRDMAQPC_AVOIDSTRETCHACK) |
- LS_64(tcp->tos, IRDMAQPC_TOS) |
- LS_64(tcp->src_port, IRDMAQPC_SRCPORTNUM) |
- LS_64(tcp->dst_port, IRDMAQPC_DESTPORTNUM);
+ qw3 |= FIELD_PREP(IRDMAQPC_TTL, tcp->ttl) |
+ FIELD_PREP(IRDMAQPC_AVOIDSTRETCHACK, tcp->avoid_stretch_ack) |
+ FIELD_PREP(IRDMAQPC_TOS, tcp->tos) |
+ FIELD_PREP(IRDMAQPC_SRCPORTNUM, tcp->src_port) |
+ FIELD_PREP(IRDMAQPC_DESTPORTNUM, tcp->dst_port);
if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) {
- qw3 |= LS_64(tcp->src_mac_addr_idx,
- IRDMAQPC_GEN1_SRCMACADDRIDX);
+ qw3 |= FIELD_PREP(IRDMAQPC_GEN1_SRCMACADDRIDX, tcp->src_mac_addr_idx);
qp->src_mac_addr_idx = tcp->src_mac_addr_idx;
}
set_64bit_val(qp_ctx, IRDMA_BYTE_32,
- LS_64(tcp->dest_ip_addr[2], IRDMAQPC_DESTIPADDR2) |
- LS_64(tcp->dest_ip_addr[3], IRDMAQPC_DESTIPADDR3));
+ FIELD_PREP(IRDMAQPC_DESTIPADDR2, tcp->dest_ip_addr[2]) |
+ FIELD_PREP(IRDMAQPC_DESTIPADDR3, tcp->dest_ip_addr[3]));
set_64bit_val(qp_ctx, IRDMA_BYTE_40,
- LS_64(tcp->dest_ip_addr[0], IRDMAQPC_DESTIPADDR0) |
- LS_64(tcp->dest_ip_addr[1], IRDMAQPC_DESTIPADDR1));
+ FIELD_PREP(IRDMAQPC_DESTIPADDR0, tcp->dest_ip_addr[0]) |
+ FIELD_PREP(IRDMAQPC_DESTIPADDR1, tcp->dest_ip_addr[1]));
set_64bit_val(qp_ctx, IRDMA_BYTE_48,
- LS_64(tcp->snd_mss, IRDMAQPC_SNDMSS) |
- LS_64(tcp->syn_rst_handling, IRDMAQPC_SYN_RST_HANDLING) |
- LS_64(tcp->vlan_tag, IRDMAQPC_VLANTAG) |
- LS_64(tcp->arp_idx, IRDMAQPC_ARPIDX));
- qw7 |= LS_64(tcp->flow_label, IRDMAQPC_FLOWLABEL) |
- LS_64(tcp->wscale, IRDMAQPC_WSCALE) |
- LS_64(tcp->ignore_tcp_opt, IRDMAQPC_IGNORE_TCP_OPT) |
- LS_64(tcp->ignore_tcp_uns_opt,
- IRDMAQPC_IGNORE_TCP_UNS_OPT) |
- LS_64(tcp->tcp_state, IRDMAQPC_TCPSTATE) |
- LS_64(tcp->rcv_wscale, IRDMAQPC_RCVSCALE) |
- LS_64(tcp->snd_wscale, IRDMAQPC_SNDSCALE);
+ FIELD_PREP(IRDMAQPC_SNDMSS, tcp->snd_mss) |
+ FIELD_PREP(IRDMAQPC_SYN_RST_HANDLING, tcp->syn_rst_handling) |
+ FIELD_PREP(IRDMAQPC_VLANTAG, tcp->vlan_tag) |
+ FIELD_PREP(IRDMAQPC_ARPIDX, tcp->arp_idx));
+ qw7 |= FIELD_PREP(IRDMAQPC_FLOWLABEL, tcp->flow_label) |
+ FIELD_PREP(IRDMAQPC_WSCALE, tcp->wscale) |
+ FIELD_PREP(IRDMAQPC_IGNORE_TCP_OPT,
+ tcp->ignore_tcp_opt) |
+ FIELD_PREP(IRDMAQPC_IGNORE_TCP_UNS_OPT,
+ tcp->ignore_tcp_uns_opt) |
+ FIELD_PREP(IRDMAQPC_TCPSTATE, tcp->tcp_state) |
+ FIELD_PREP(IRDMAQPC_RCVSCALE, tcp->rcv_wscale) |
+ FIELD_PREP(IRDMAQPC_SNDSCALE, tcp->snd_wscale);
set_64bit_val(qp_ctx, IRDMA_BYTE_72,
- LS_64(tcp->time_stamp_recent, IRDMAQPC_TIMESTAMP_RECENT) |
- LS_64(tcp->time_stamp_age, IRDMAQPC_TIMESTAMP_AGE));
+ FIELD_PREP(IRDMAQPC_TIMESTAMP_RECENT, tcp->time_stamp_recent) |
+ FIELD_PREP(IRDMAQPC_TIMESTAMP_AGE, tcp->time_stamp_age));
set_64bit_val(qp_ctx, IRDMA_BYTE_80,
- LS_64(tcp->snd_nxt, IRDMAQPC_SNDNXT) |
- LS_64(tcp->snd_wnd, IRDMAQPC_SNDWND));
+ FIELD_PREP(IRDMAQPC_SNDNXT, tcp->snd_nxt) |
+ FIELD_PREP(IRDMAQPC_SNDWND, tcp->snd_wnd));
set_64bit_val(qp_ctx, IRDMA_BYTE_88,
- LS_64(tcp->rcv_nxt, IRDMAQPC_RCVNXT) |
- LS_64(tcp->rcv_wnd, IRDMAQPC_RCVWND));
+ FIELD_PREP(IRDMAQPC_RCVNXT, tcp->rcv_nxt) |
+ FIELD_PREP(IRDMAQPC_RCVWND, tcp->rcv_wnd));
set_64bit_val(qp_ctx, IRDMA_BYTE_96,
- LS_64(tcp->snd_max, IRDMAQPC_SNDMAX) |
- LS_64(tcp->snd_una, IRDMAQPC_SNDUNA));
+ FIELD_PREP(IRDMAQPC_SNDMAX, tcp->snd_max) |
+ FIELD_PREP(IRDMAQPC_SNDUNA, tcp->snd_una));
set_64bit_val(qp_ctx, IRDMA_BYTE_104,
- LS_64(tcp->srtt, IRDMAQPC_SRTT) |
- LS_64(tcp->rtt_var, IRDMAQPC_RTTVAR));
+ FIELD_PREP(IRDMAQPC_SRTT, tcp->srtt) |
+ FIELD_PREP(IRDMAQPC_RTTVAR, tcp->rtt_var));
set_64bit_val(qp_ctx, IRDMA_BYTE_112,
- LS_64(tcp->ss_thresh, IRDMAQPC_SSTHRESH) |
- LS_64(tcp->cwnd, IRDMAQPC_CWND));
+ FIELD_PREP(IRDMAQPC_SSTHRESH, tcp->ss_thresh) |
+ FIELD_PREP(IRDMAQPC_CWND, tcp->cwnd));
set_64bit_val(qp_ctx, IRDMA_BYTE_120,
- LS_64(tcp->snd_wl1, IRDMAQPC_SNDWL1) |
- LS_64(tcp->snd_wl2, IRDMAQPC_SNDWL2));
- qw16 |= LS_64(tcp->max_snd_window, IRDMAQPC_MAXSNDWND) |
- LS_64(tcp->rexmit_thresh, IRDMAQPC_REXMIT_THRESH);
+ FIELD_PREP(IRDMAQPC_SNDWL1, tcp->snd_wl1) |
+ FIELD_PREP(IRDMAQPC_SNDWL2, tcp->snd_wl2));
+ qw16 |= FIELD_PREP(IRDMAQPC_MAXSNDWND, tcp->max_snd_window) |
+ FIELD_PREP(IRDMAQPC_REXMIT_THRESH, tcp->rexmit_thresh);
set_64bit_val(qp_ctx, IRDMA_BYTE_184,
- LS_64(tcp->local_ipaddr[3], IRDMAQPC_LOCAL_IPADDR3) |
- LS_64(tcp->local_ipaddr[2], IRDMAQPC_LOCAL_IPADDR2));
+ FIELD_PREP(IRDMAQPC_LOCAL_IPADDR3, tcp->local_ipaddr[3]) |
+ FIELD_PREP(IRDMAQPC_LOCAL_IPADDR2, tcp->local_ipaddr[2]));
set_64bit_val(qp_ctx, IRDMA_BYTE_192,
- LS_64(tcp->local_ipaddr[1], IRDMAQPC_LOCAL_IPADDR1) |
- LS_64(tcp->local_ipaddr[0], IRDMAQPC_LOCAL_IPADDR0));
+ FIELD_PREP(IRDMAQPC_LOCAL_IPADDR1, tcp->local_ipaddr[1]) |
+ FIELD_PREP(IRDMAQPC_LOCAL_IPADDR0, tcp->local_ipaddr[0]));
set_64bit_val(qp_ctx, IRDMA_BYTE_200,
- LS_64(iw->t_high, IRDMAQPC_THIGH) |
- LS_64(iw->t_low, IRDMAQPC_TLOW));
+ FIELD_PREP(IRDMAQPC_THIGH, iw->t_high) |
+ FIELD_PREP(IRDMAQPC_TLOW, iw->t_low));
set_64bit_val(qp_ctx, IRDMA_BYTE_208,
- LS_64(info->rem_endpoint_idx, IRDMAQPC_REMENDPOINTIDX));
+ FIELD_PREP(IRDMAQPC_REMENDPOINTIDX, info->rem_endpoint_idx));
}
set_64bit_val(qp_ctx, IRDMA_BYTE_0, qw0);
@@ -1145,6 +1155,9 @@
u64 hdr;
enum irdma_page_size page_size;
+ if (!info->total_len && !info->all_memory)
+ return -EINVAL;
+
if (info->page_size == 0x40000000)
page_size = IRDMA_PAGE_SIZE_1G;
else if (info->page_size == 0x200000)
@@ -1159,25 +1172,25 @@
set_64bit_val(wqe, IRDMA_BYTE_8,
FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID) |
- LS_64(info->total_len, IRDMA_CQPSQ_STAG_STAGLEN));
+ FIELD_PREP(IRDMA_CQPSQ_STAG_STAGLEN, info->total_len));
set_64bit_val(wqe, IRDMA_BYTE_16,
- LS_64(info->stag_idx, IRDMA_CQPSQ_STAG_IDX));
+ FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info->stag_idx));
set_64bit_val(wqe, IRDMA_BYTE_40,
- LS_64(info->hmc_fcn_index, IRDMA_CQPSQ_STAG_HMCFNIDX));
+ FIELD_PREP(IRDMA_CQPSQ_STAG_HMCFNIDX, info->hmc_fcn_index));
if (info->chunk_size)
set_64bit_val(wqe, IRDMA_BYTE_48,
- LS_64(info->first_pm_pbl_idx, IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX));
-
- hdr = LS_64(IRDMA_CQP_OP_ALLOC_STAG, IRDMA_CQPSQ_OPCODE) |
- LS_64(1, IRDMA_CQPSQ_STAG_MR) |
- LS_64(info->access_rights, IRDMA_CQPSQ_STAG_ARIGHTS) |
- LS_64(info->chunk_size, IRDMA_CQPSQ_STAG_LPBLSIZE) |
- LS_64(page_size, IRDMA_CQPSQ_STAG_HPAGESIZE) |
- LS_64(info->remote_access, IRDMA_CQPSQ_STAG_REMACCENABLED) |
- LS_64(info->use_hmc_fcn_index, IRDMA_CQPSQ_STAG_USEHMCFNIDX) |
- LS_64(info->use_pf_rid, IRDMA_CQPSQ_STAG_USEPFRID) |
- LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ FIELD_PREP(IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX, info->first_pm_pbl_idx));
+
+ hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_ALLOC_STAG) |
+ FIELD_PREP(IRDMA_CQPSQ_STAG_MR, 1) |
+ FIELD_PREP(IRDMA_CQPSQ_STAG_ARIGHTS, info->access_rights) |
+ FIELD_PREP(IRDMA_CQPSQ_STAG_LPBLSIZE, info->chunk_size) |
+ FIELD_PREP(IRDMA_CQPSQ_STAG_HPAGESIZE, page_size) |
+ FIELD_PREP(IRDMA_CQPSQ_STAG_REMACCENABLED, info->remote_access) |
+ FIELD_PREP(IRDMA_CQPSQ_STAG_USEHMCFNIDX, info->use_hmc_fcn_index) |
+ FIELD_PREP(IRDMA_CQPSQ_STAG_USEPFRID, info->use_pf_rid) |
+ FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
@@ -1211,6 +1224,9 @@
u8 addr_type;
enum irdma_page_size page_size;
+ if (!info->total_len && !info->all_memory)
+ return -EINVAL;
+
if (info->page_size == 0x40000000)
page_size = IRDMA_PAGE_SIZE_1G;
else if (info->page_size == 0x200000)
@@ -1240,30 +1256,30 @@
(info->addr_type == IRDMA_ADDR_TYPE_VA_BASED ?
info->va : fbo));
set_64bit_val(wqe, IRDMA_BYTE_8,
- LS_64(info->total_len, IRDMA_CQPSQ_STAG_STAGLEN) |
+ FIELD_PREP(IRDMA_CQPSQ_STAG_STAGLEN, info->total_len) |
FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID));
set_64bit_val(wqe, IRDMA_BYTE_16,
- LS_64(info->stag_key, IRDMA_CQPSQ_STAG_KEY) |
- LS_64(info->stag_idx, IRDMA_CQPSQ_STAG_IDX));
+ FIELD_PREP(IRDMA_CQPSQ_STAG_KEY, info->stag_key) |
+ FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info->stag_idx));
if (!info->chunk_size)
set_64bit_val(wqe, IRDMA_BYTE_32, info->reg_addr_pa);
else
set_64bit_val(wqe, IRDMA_BYTE_48,
- LS_64(info->first_pm_pbl_index, IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX));
+ FIELD_PREP(IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX, info->first_pm_pbl_index));
set_64bit_val(wqe, IRDMA_BYTE_40, info->hmc_fcn_index);
addr_type = (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED) ? 1 : 0;
- hdr = LS_64(IRDMA_CQP_OP_REG_MR, IRDMA_CQPSQ_OPCODE) |
- LS_64(1, IRDMA_CQPSQ_STAG_MR) |
- LS_64(info->chunk_size, IRDMA_CQPSQ_STAG_LPBLSIZE) |
- LS_64(page_size, IRDMA_CQPSQ_STAG_HPAGESIZE) |
- LS_64(info->access_rights, IRDMA_CQPSQ_STAG_ARIGHTS) |
- LS_64(remote_access, IRDMA_CQPSQ_STAG_REMACCENABLED) |
- LS_64(addr_type, IRDMA_CQPSQ_STAG_VABASEDTO) |
- LS_64(info->use_hmc_fcn_index, IRDMA_CQPSQ_STAG_USEHMCFNIDX) |
- LS_64(info->use_pf_rid, IRDMA_CQPSQ_STAG_USEPFRID) |
- LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_REG_MR) |
+ FIELD_PREP(IRDMA_CQPSQ_STAG_MR, 1) |
+ FIELD_PREP(IRDMA_CQPSQ_STAG_LPBLSIZE, info->chunk_size) |
+ FIELD_PREP(IRDMA_CQPSQ_STAG_HPAGESIZE, page_size) |
+ FIELD_PREP(IRDMA_CQPSQ_STAG_ARIGHTS, info->access_rights) |
+ FIELD_PREP(IRDMA_CQPSQ_STAG_REMACCENABLED, remote_access) |
+ FIELD_PREP(IRDMA_CQPSQ_STAG_VABASEDTO, addr_type) |
+ FIELD_PREP(IRDMA_CQPSQ_STAG_USEHMCFNIDX, info->use_hmc_fcn_index) |
+ FIELD_PREP(IRDMA_CQPSQ_STAG_USEPFRID, info->use_pf_rid) |
+ FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
@@ -1300,11 +1316,11 @@
set_64bit_val(wqe, IRDMA_BYTE_8,
FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID));
set_64bit_val(wqe, IRDMA_BYTE_16,
- LS_64(info->stag_idx, IRDMA_CQPSQ_STAG_IDX));
+ FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info->stag_idx));
- hdr = LS_64(IRDMA_CQP_OP_DEALLOC_STAG, IRDMA_CQPSQ_OPCODE) |
- LS_64(info->mr, IRDMA_CQPSQ_STAG_MR) |
- LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DEALLOC_STAG) |
+ FIELD_PREP(IRDMA_CQPSQ_STAG_MR, info->mr) |
+ FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
@@ -1341,13 +1357,13 @@
set_64bit_val(wqe, IRDMA_BYTE_8,
FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID));
set_64bit_val(wqe, IRDMA_BYTE_16,
- LS_64(info->mw_stag_index, IRDMA_CQPSQ_STAG_IDX));
+ FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info->mw_stag_index));
- hdr = LS_64(IRDMA_CQP_OP_ALLOC_STAG, IRDMA_CQPSQ_OPCODE) |
- LS_64(info->mw_wide, IRDMA_CQPSQ_STAG_MWTYPE) |
- LS_64(info->mw1_bind_dont_vldt_key,
- IRDMA_CQPSQ_STAG_MW1_BIND_DONT_VLDT_KEY) |
- LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_ALLOC_STAG) |
+ FIELD_PREP(IRDMA_CQPSQ_STAG_MWTYPE, info->mw_wide) |
+ FIELD_PREP(IRDMA_CQPSQ_STAG_MW1_BIND_DONT_VLDT_KEY,
+ info->mw1_bind_dont_vldt_key) |
+ FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
@@ -1374,6 +1390,7 @@
u64 temp, hdr;
__le64 *wqe;
u32 wqe_idx;
+ u16 quanta = IRDMA_QP_WQE_MIN_QUANTA;
enum irdma_page_size page_size;
struct irdma_post_sq_info sq_info = {0};
@@ -1388,13 +1405,10 @@
sq_info.signaled = info->signaled;
sq_info.push_wqe = info->push_wqe;
- wqe = irdma_qp_get_next_send_wqe(&qp->qp_uk, &wqe_idx,
- IRDMA_QP_WQE_MIN_QUANTA, 0, &sq_info);
+ wqe = irdma_qp_get_next_send_wqe(&qp->qp_uk, &wqe_idx, &quanta, 0, &sq_info);
if (!wqe)
return -ENOSPC;
- irdma_clr_wqes(&qp->qp_uk, wqe_idx);
-
qp->qp_uk.sq_wrtrk_array[wqe_idx].signaled = info->signaled;
irdma_debug(qp->dev, IRDMA_DEBUG_MR,
"wr_id[%llxh] wqe_idx[%04d] location[%p]\n", (unsigned long long)info->wr_id,
@@ -1404,39 +1418,37 @@
(uintptr_t)info->va : info->fbo;
set_64bit_val(wqe, IRDMA_BYTE_0, temp);
- temp = RS_64(info->first_pm_pbl_index >> 16, IRDMAQPSQ_FIRSTPMPBLIDXHI);
+ temp = FIELD_GET(IRDMAQPSQ_FIRSTPMPBLIDXHI,
+ info->first_pm_pbl_index >> 16);
set_64bit_val(wqe, IRDMA_BYTE_8,
- LS_64(temp, IRDMAQPSQ_FIRSTPMPBLIDXHI) |
- LS_64(info->reg_addr_pa >> IRDMAQPSQ_PBLADDR_S, IRDMAQPSQ_PBLADDR));
+ FIELD_PREP(IRDMAQPSQ_FIRSTPMPBLIDXHI, temp) |
+ FIELD_PREP(IRDMAQPSQ_PBLADDR, info->reg_addr_pa >> IRDMA_HW_PAGE_SHIFT));
set_64bit_val(wqe, IRDMA_BYTE_16,
info->total_len |
- LS_64(info->first_pm_pbl_index, IRDMAQPSQ_FIRSTPMPBLIDXLO));
-
- hdr = LS_64(info->stag_key, IRDMAQPSQ_STAGKEY) |
- LS_64(info->stag_idx, IRDMAQPSQ_STAGINDEX) |
- LS_64(IRDMAQP_OP_FAST_REGISTER, IRDMAQPSQ_OPCODE) |
- LS_64(info->chunk_size, IRDMAQPSQ_LPBLSIZE) |
- LS_64(page_size, IRDMAQPSQ_HPAGESIZE) |
- LS_64(info->access_rights, IRDMAQPSQ_STAGRIGHTS) |
- LS_64(info->addr_type, IRDMAQPSQ_VABASEDTO) |
- LS_64((sq_info.push_wqe ? 1 : 0), IRDMAQPSQ_PUSHWQE) |
- LS_64(info->read_fence, IRDMAQPSQ_READFENCE) |
- LS_64(info->local_fence, IRDMAQPSQ_LOCALFENCE) |
- LS_64(info->signaled, IRDMAQPSQ_SIGCOMPL) |
- LS_64(qp->qp_uk.swqe_polarity, IRDMAQPSQ_VALID);
+ FIELD_PREP(IRDMAQPSQ_FIRSTPMPBLIDXLO, info->first_pm_pbl_index));
+
+ hdr = FIELD_PREP(IRDMAQPSQ_STAGKEY, info->stag_key) |
+ FIELD_PREP(IRDMAQPSQ_STAGINDEX, info->stag_idx) |
+ FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_FAST_REGISTER) |
+ FIELD_PREP(IRDMAQPSQ_LPBLSIZE, info->chunk_size) |
+ FIELD_PREP(IRDMAQPSQ_HPAGESIZE, page_size) |
+ FIELD_PREP(IRDMAQPSQ_STAGRIGHTS, info->access_rights) |
+ FIELD_PREP(IRDMAQPSQ_VABASEDTO, info->addr_type) |
+ FIELD_PREP(IRDMAQPSQ_PUSHWQE, (sq_info.push_wqe ? 1 : 0)) |
+ FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) |
+ FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) |
+ FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
irdma_debug_buf(qp->dev, IRDMA_DEBUG_WQE, "FAST_REG WQE", wqe,
IRDMA_QP_WQE_MIN_SIZE);
- if (sq_info.push_wqe) {
- irdma_qp_push_wqe(&qp->qp_uk, wqe, IRDMA_QP_WQE_MIN_QUANTA,
- wqe_idx, post_sq);
- } else {
- if (post_sq)
- irdma_uk_qp_post_wr(&qp->qp_uk);
- }
+ if (sq_info.push_wqe)
+ irdma_qp_push_wqe(&qp->qp_uk, wqe, quanta, wqe_idx, post_sq);
+ else if (post_sq)
+ irdma_uk_qp_post_wr(&qp->qp_uk);
return 0;
}
@@ -1456,9 +1468,9 @@
wqe = qp_uk->sq_base[1].elem;
- hdr = LS_64(IRDMAQP_OP_NOP, IRDMAQPSQ_OPCODE) |
- LS_64(1, IRDMAQPSQ_LOCALFENCE) |
- LS_64(qp->qp_uk.swqe_polarity, IRDMAQPSQ_VALID);
+ hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) |
+ FIELD_PREP(IRDMAQPSQ_LOCALFENCE, 1) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
@@ -1466,8 +1478,8 @@
IRDMA_QP_WQE_MIN_SIZE);
wqe = qp_uk->sq_base[2].elem;
- hdr = LS_64(IRDMAQP_OP_GEN_RTS_AE, IRDMAQPSQ_OPCODE) |
- LS_64(qp->qp_uk.swqe_polarity, IRDMAQPSQ_VALID);
+ hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_GEN_RTS_AE) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
@@ -1482,7 +1494,7 @@
* @size: size of lsmm buffer
* @stag: stag of lsmm buffer
*/
-int
+void
irdma_sc_send_lsmm(struct irdma_sc_qp *qp, void *lsmm_buf, u32 size,
irdma_stag stag)
{
@@ -1496,20 +1508,20 @@
set_64bit_val(wqe, IRDMA_BYTE_0, (uintptr_t)lsmm_buf);
if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) {
set_64bit_val(wqe, IRDMA_BYTE_8,
- LS_64(size, IRDMAQPSQ_GEN1_FRAG_LEN) |
- LS_64(stag, IRDMAQPSQ_GEN1_FRAG_STAG));
+ FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, size) |
+ FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, stag));
} else {
set_64bit_val(wqe, IRDMA_BYTE_8,
- LS_64(size, IRDMAQPSQ_FRAG_LEN) |
- LS_64(stag, IRDMAQPSQ_FRAG_STAG) |
- LS_64(qp->qp_uk.swqe_polarity, IRDMAQPSQ_VALID));
+ FIELD_PREP(IRDMAQPSQ_FRAG_LEN, size) |
+ FIELD_PREP(IRDMAQPSQ_FRAG_STAG, stag) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity));
}
set_64bit_val(wqe, IRDMA_BYTE_16, 0);
- hdr = LS_64(IRDMAQP_OP_RDMA_SEND, IRDMAQPSQ_OPCODE) |
- LS_64(1, IRDMAQPSQ_STREAMMODE) |
- LS_64(1, IRDMAQPSQ_WAITFORRCVPDU) |
- LS_64(qp->qp_uk.swqe_polarity, IRDMAQPSQ_VALID);
+ hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_RDMA_SEND) |
+ FIELD_PREP(IRDMAQPSQ_STREAMMODE, 1) |
+ FIELD_PREP(IRDMAQPSQ_WAITFORRCVPDU, 1) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
@@ -1519,8 +1531,6 @@
if (qp->dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE)
irdma_sc_gen_rts_ae(qp);
-
- return 0;
}
/**
@@ -1529,7 +1539,7 @@
* @lsmm_buf: buffer with lsmm message
* @size: size of lsmm buffer
*/
-int
+void
irdma_sc_send_lsmm_nostag(struct irdma_sc_qp *qp, void *lsmm_buf, u32 size)
{
__le64 *wqe;
@@ -1543,25 +1553,23 @@
if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1)
set_64bit_val(wqe, IRDMA_BYTE_8,
- LS_64(size, IRDMAQPSQ_GEN1_FRAG_LEN));
+ FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, size));
else
set_64bit_val(wqe, IRDMA_BYTE_8,
- LS_64(size, IRDMAQPSQ_FRAG_LEN) |
- LS_64(qp->qp_uk.swqe_polarity, IRDMAQPSQ_VALID));
+ FIELD_PREP(IRDMAQPSQ_FRAG_LEN, size) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity));
set_64bit_val(wqe, IRDMA_BYTE_16, 0);
- hdr = LS_64(IRDMAQP_OP_RDMA_SEND, IRDMAQPSQ_OPCODE) |
- LS_64(1, IRDMAQPSQ_STREAMMODE) |
- LS_64(1, IRDMAQPSQ_WAITFORRCVPDU) |
- LS_64(qp->qp_uk.swqe_polarity, IRDMAQPSQ_VALID);
+ hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_RDMA_SEND) |
+ FIELD_PREP(IRDMAQPSQ_STREAMMODE, 1) |
+ FIELD_PREP(IRDMAQPSQ_WAITFORRCVPDU, 1) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
irdma_debug_buf(qp->dev, IRDMA_DEBUG_WQE, "SEND_LSMM_NOSTAG WQE", wqe,
IRDMA_QP_WQE_MIN_SIZE);
-
- return 0;
}
/**
@@ -1569,7 +1577,7 @@
* @qp: sc qp struct
* @read: Do read0 or write0
*/
-int
+void
irdma_sc_send_rtt(struct irdma_sc_qp *qp, bool read)
{
__le64 *wqe;
@@ -1584,26 +1592,25 @@
if (read) {
if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) {
set_64bit_val(wqe, IRDMA_BYTE_8,
- LS_64(0xabcd, IRDMAQPSQ_GEN1_FRAG_STAG));
+ FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, 0xabcd));
} else {
set_64bit_val(wqe, IRDMA_BYTE_8,
- (u64)0xabcd | LS_64(qp->qp_uk.swqe_polarity,
- IRDMAQPSQ_VALID));
+ (u64)0xabcd | FIELD_PREP(IRDMAQPSQ_VALID,
+ qp->qp_uk.swqe_polarity));
}
- hdr = LS_64(0x1234, IRDMAQPSQ_REMSTAG) |
- LS_64(IRDMAQP_OP_RDMA_READ, IRDMAQPSQ_OPCODE) |
- LS_64(qp->qp_uk.swqe_polarity, IRDMAQPSQ_VALID);
+ hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, 0x1234) |
+ FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_RDMA_READ) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);
} else {
if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) {
set_64bit_val(wqe, IRDMA_BYTE_8, 0);
} else {
set_64bit_val(wqe, IRDMA_BYTE_8,
- LS_64(qp->qp_uk.swqe_polarity,
- IRDMAQPSQ_VALID));
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity));
}
- hdr = LS_64(IRDMAQP_OP_RDMA_WRITE, IRDMAQPSQ_OPCODE) |
- LS_64(qp->qp_uk.swqe_polarity, IRDMAQPSQ_VALID);
+ hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_RDMA_WRITE) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);
}
irdma_wmb(); /* make sure WQE is written before valid bit is set */
@@ -1615,8 +1622,6 @@
if (qp->dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE)
irdma_sc_gen_rts_ae(qp);
-
- return 0;
}
/**
@@ -2009,8 +2014,6 @@
vsi->mtu = info->params->mtu;
vsi->exception_lan_q = info->exception_lan_q;
vsi->vsi_idx = info->pf_data_vsi_num;
- if (vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
- vsi->fcn_id = info->dev->hmc_fn_id;
irdma_set_qos_info(vsi, info->params);
for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) {
@@ -2029,31 +2032,20 @@
}
/**
- * irdma_get_fcn_id - Return the function id
+ * irdma_get_stats_idx - Return stats index
* @vsi: pointer to the vsi
*/
-static u8 irdma_get_fcn_id(struct irdma_sc_vsi *vsi){
+static u8 irdma_get_stats_idx(struct irdma_sc_vsi *vsi){
struct irdma_stats_inst_info stats_info = {0};
struct irdma_sc_dev *dev = vsi->dev;
- u8 fcn_id = IRDMA_INVALID_FCN_ID;
- u8 start_idx, max_stats, i;
- if (dev->hw_attrs.uk_attrs.hw_rev != IRDMA_GEN_1) {
+ if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
if (!irdma_cqp_stats_inst_cmd(vsi, IRDMA_OP_STATS_ALLOCATE,
&stats_info))
return stats_info.stats_idx;
}
- start_idx = 1;
- max_stats = 16;
- for (i = start_idx; i < max_stats; i++)
- if (!dev->fcn_id_array[i]) {
- fcn_id = i;
- dev->fcn_id_array[i] = true;
- break;
- }
-
- return fcn_id;
+ return IRDMA_INVALID_STATS_IDX;
}
/**
@@ -2065,12 +2057,12 @@
irdma_vsi_stats_init(struct irdma_sc_vsi *vsi,
struct irdma_vsi_stats_info *info)
{
- u8 fcn_id = info->fcn_id;
struct irdma_dma_mem *stats_buff_mem;
vsi->pestat = info->pestat;
vsi->pestat->hw = vsi->dev->hw;
vsi->pestat->vsi = vsi;
+
stats_buff_mem = &vsi->pestat->gather_info.stats_buff_mem;
stats_buff_mem->size = IRDMA_GATHER_STATS_BUF_SIZE * 2;
stats_buff_mem->va = irdma_allocate_dma_mem(vsi->pestat->hw,
@@ -2085,24 +2077,21 @@
IRDMA_GATHER_STATS_BUF_SIZE);
irdma_hw_stats_start_timer(vsi);
- if (info->alloc_fcn_id)
- fcn_id = irdma_get_fcn_id(vsi);
- if (fcn_id == IRDMA_INVALID_FCN_ID)
- goto stats_error;
-
- vsi->stats_fcn_id_alloc = info->alloc_fcn_id;
- vsi->fcn_id = fcn_id;
- if (info->alloc_fcn_id) {
- vsi->pestat->gather_info.use_stats_inst = true;
- vsi->pestat->gather_info.stats_inst_index = fcn_id;
- }
- return 0;
+ /* when stat allocation is not required default to fcn_id. */
+ vsi->stats_idx = info->fcn_id;
+ if (info->alloc_stats_inst) {
+ u8 stats_idx = irdma_get_stats_idx(vsi);
-stats_error:
- irdma_free_dma_mem(vsi->pestat->hw, stats_buff_mem);
+ if (stats_idx != IRDMA_INVALID_STATS_IDX) {
+ vsi->stats_inst_alloc = true;
+ vsi->stats_idx = stats_idx;
+ vsi->pestat->gather_info.use_stats_inst = true;
+ vsi->pestat->gather_info.stats_inst_index = stats_idx;
+ }
+ }
- return -EIO;
+ return 0;
}
/**
@@ -2113,23 +2102,19 @@
irdma_vsi_stats_free(struct irdma_sc_vsi *vsi)
{
struct irdma_stats_inst_info stats_info = {0};
- u8 fcn_id = vsi->fcn_id;
struct irdma_sc_dev *dev = vsi->dev;
- if (dev->hw_attrs.uk_attrs.hw_rev != IRDMA_GEN_1) {
- if (vsi->stats_fcn_id_alloc) {
- stats_info.stats_idx = vsi->fcn_id;
+ if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
+ if (vsi->stats_inst_alloc) {
+ stats_info.stats_idx = vsi->stats_idx;
irdma_cqp_stats_inst_cmd(vsi, IRDMA_OP_STATS_FREE,
&stats_info);
}
- } else {
- if (vsi->stats_fcn_id_alloc &&
- fcn_id < vsi->dev->hw_attrs.max_stat_inst)
- vsi->dev->fcn_id_array[fcn_id] = false;
}
if (!vsi->pestat)
return;
+
irdma_hw_stats_stop_timer(vsi);
irdma_free_dma_mem(vsi->pestat->hw,
&vsi->pestat->gather_info.stats_buff_mem);
@@ -2179,15 +2164,16 @@
return -ENOSPC;
set_64bit_val(wqe, IRDMA_BYTE_40,
- LS_64(info->hmc_fcn_index, IRDMA_CQPSQ_STATS_HMC_FCN_INDEX));
+ FIELD_PREP(IRDMA_CQPSQ_STATS_HMC_FCN_INDEX, info->hmc_fcn_index));
set_64bit_val(wqe, IRDMA_BYTE_32, info->stats_buff_mem.pa);
- temp = LS_64(cqp->polarity, IRDMA_CQPSQ_STATS_WQEVALID) |
- LS_64(info->use_stats_inst, IRDMA_CQPSQ_STATS_USE_INST) |
- LS_64(info->stats_inst_index, IRDMA_CQPSQ_STATS_INST_INDEX) |
- LS_64(info->use_hmc_fcn_index,
- IRDMA_CQPSQ_STATS_USE_HMC_FCN_INDEX) |
- LS_64(IRDMA_CQP_OP_GATHER_STATS, IRDMA_CQPSQ_STATS_OP);
+ temp = FIELD_PREP(IRDMA_CQPSQ_STATS_WQEVALID, cqp->polarity) |
+ FIELD_PREP(IRDMA_CQPSQ_STATS_USE_INST, info->use_stats_inst) |
+ FIELD_PREP(IRDMA_CQPSQ_STATS_INST_INDEX,
+ info->stats_inst_index) |
+ FIELD_PREP(IRDMA_CQPSQ_STATS_USE_HMC_FCN_INDEX,
+ info->use_hmc_fcn_index) |
+ FIELD_PREP(IRDMA_CQPSQ_STATS_OP, IRDMA_CQP_OP_GATHER_STATS);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, temp);
@@ -2223,12 +2209,13 @@
return -ENOSPC;
set_64bit_val(wqe, IRDMA_BYTE_40,
- LS_64(info->hmc_fn_id, IRDMA_CQPSQ_STATS_HMC_FCN_INDEX));
- temp = LS_64(cqp->polarity, IRDMA_CQPSQ_STATS_WQEVALID) |
- LS_64(alloc, IRDMA_CQPSQ_STATS_ALLOC_INST) |
- LS_64(info->use_hmc_fcn_index, IRDMA_CQPSQ_STATS_USE_HMC_FCN_INDEX) |
- LS_64(info->stats_idx, IRDMA_CQPSQ_STATS_INST_INDEX) |
- LS_64(IRDMA_CQP_OP_MANAGE_STATS, IRDMA_CQPSQ_STATS_OP);
+ FIELD_PREP(IRDMA_CQPSQ_STATS_HMC_FCN_INDEX, info->hmc_fn_id));
+ temp = FIELD_PREP(IRDMA_CQPSQ_STATS_WQEVALID, cqp->polarity) |
+ FIELD_PREP(IRDMA_CQPSQ_STATS_ALLOC_INST, alloc) |
+ FIELD_PREP(IRDMA_CQPSQ_STATS_USE_HMC_FCN_INDEX,
+ info->use_hmc_fcn_index) |
+ FIELD_PREP(IRDMA_CQPSQ_STATS_INST_INDEX, info->stats_idx) |
+ FIELD_PREP(IRDMA_CQPSQ_STATS_OP, IRDMA_CQP_OP_MANAGE_STATS);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
@@ -2265,13 +2252,14 @@
set_64bit_val(wqe, IRDMA_BYTE_0, temp);
set_64bit_val(wqe, IRDMA_BYTE_40,
- LS_64(info->cnp_up_override, IRDMA_CQPSQ_UP_CNPOVERRIDE) |
- LS_64(info->hmc_fcn_idx, IRDMA_CQPSQ_UP_HMCFCNIDX));
-
- temp = LS_64(cqp->polarity, IRDMA_CQPSQ_UP_WQEVALID) |
- LS_64(info->use_vlan, IRDMA_CQPSQ_UP_USEVLAN) |
- LS_64(info->use_cnp_up_override, IRDMA_CQPSQ_UP_USEOVERRIDE) |
- LS_64(IRDMA_CQP_OP_UP_MAP, IRDMA_CQPSQ_UP_OP);
+ FIELD_PREP(IRDMA_CQPSQ_UP_CNPOVERRIDE, info->cnp_up_override) |
+ FIELD_PREP(IRDMA_CQPSQ_UP_HMCFCNIDX, info->hmc_fcn_idx));
+
+ temp = FIELD_PREP(IRDMA_CQPSQ_UP_WQEVALID, cqp->polarity) |
+ FIELD_PREP(IRDMA_CQPSQ_UP_USEVLAN, info->use_vlan) |
+ FIELD_PREP(IRDMA_CQPSQ_UP_USEOVERRIDE,
+ info->use_cnp_up_override) |
+ FIELD_PREP(IRDMA_CQPSQ_UP_OP, IRDMA_CQP_OP_UP_MAP);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, temp);
@@ -2303,18 +2291,18 @@
return -ENOSPC;
set_64bit_val(wqe, IRDMA_BYTE_32,
- LS_64(info->vsi, IRDMA_CQPSQ_WS_VSI) |
- LS_64(info->weight, IRDMA_CQPSQ_WS_WEIGHT));
-
- temp = LS_64(cqp->polarity, IRDMA_CQPSQ_WS_WQEVALID) |
- LS_64(node_op, IRDMA_CQPSQ_WS_NODEOP) |
- LS_64(info->enable, IRDMA_CQPSQ_WS_ENABLENODE) |
- LS_64(info->type_leaf, IRDMA_CQPSQ_WS_NODETYPE) |
- LS_64(info->prio_type, IRDMA_CQPSQ_WS_PRIOTYPE) |
- LS_64(info->tc, IRDMA_CQPSQ_WS_TC) |
- LS_64(IRDMA_CQP_OP_WORK_SCHED_NODE, IRDMA_CQPSQ_WS_OP) |
- LS_64(info->parent_id, IRDMA_CQPSQ_WS_PARENTID) |
- LS_64(info->id, IRDMA_CQPSQ_WS_NODEID);
+ FIELD_PREP(IRDMA_CQPSQ_WS_VSI, info->vsi) |
+ FIELD_PREP(IRDMA_CQPSQ_WS_WEIGHT, info->weight));
+
+ temp = FIELD_PREP(IRDMA_CQPSQ_WS_WQEVALID, cqp->polarity) |
+ FIELD_PREP(IRDMA_CQPSQ_WS_NODEOP, node_op) |
+ FIELD_PREP(IRDMA_CQPSQ_WS_ENABLENODE, info->enable) |
+ FIELD_PREP(IRDMA_CQPSQ_WS_NODETYPE, info->type_leaf) |
+ FIELD_PREP(IRDMA_CQPSQ_WS_PRIOTYPE, info->prio_type) |
+ FIELD_PREP(IRDMA_CQPSQ_WS_TC, info->tc) |
+ FIELD_PREP(IRDMA_CQPSQ_WS_OP, IRDMA_CQP_OP_WORK_SCHED_NODE) |
+ FIELD_PREP(IRDMA_CQPSQ_WS_PARENTID, info->parent_id) |
+ FIELD_PREP(IRDMA_CQPSQ_WS_NODEID, info->id);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, temp);
@@ -2364,25 +2352,30 @@
if (info->userflushcode) {
if (flush_rq)
- temp |= LS_64(info->rq_minor_code, IRDMA_CQPSQ_FWQE_RQMNERR) |
- LS_64(info->rq_major_code, IRDMA_CQPSQ_FWQE_RQMJERR);
+ temp |= FIELD_PREP(IRDMA_CQPSQ_FWQE_RQMNERR,
+ info->rq_minor_code) |
+ FIELD_PREP(IRDMA_CQPSQ_FWQE_RQMJERR,
+ info->rq_major_code);
if (flush_sq)
- temp |= LS_64(info->sq_minor_code, IRDMA_CQPSQ_FWQE_SQMNERR) |
- LS_64(info->sq_major_code, IRDMA_CQPSQ_FWQE_SQMJERR);
+ temp |= FIELD_PREP(IRDMA_CQPSQ_FWQE_SQMNERR,
+ info->sq_minor_code) |
+ FIELD_PREP(IRDMA_CQPSQ_FWQE_SQMJERR,
+ info->sq_major_code);
}
set_64bit_val(wqe, IRDMA_BYTE_16, temp);
temp = (info->generate_ae) ?
- info->ae_code | LS_64(info->ae_src, IRDMA_CQPSQ_FWQE_AESOURCE) : 0;
+ info->ae_code | FIELD_PREP(IRDMA_CQPSQ_FWQE_AESOURCE,
+ info->ae_src) : 0;
set_64bit_val(wqe, IRDMA_BYTE_8, temp);
hdr = qp->qp_uk.qp_id |
- LS_64(IRDMA_CQP_OP_FLUSH_WQES, IRDMA_CQPSQ_OPCODE) |
- LS_64(info->generate_ae, IRDMA_CQPSQ_FWQE_GENERATE_AE) |
- LS_64(info->userflushcode, IRDMA_CQPSQ_FWQE_USERFLCODE) |
- LS_64(flush_sq, IRDMA_CQPSQ_FWQE_FLUSHSQ) |
- LS_64(flush_rq, IRDMA_CQPSQ_FWQE_FLUSHRQ) |
- LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_FLUSH_WQES) |
+ FIELD_PREP(IRDMA_CQPSQ_FWQE_GENERATE_AE, info->generate_ae) |
+ FIELD_PREP(IRDMA_CQPSQ_FWQE_USERFLCODE, info->userflushcode) |
+ FIELD_PREP(IRDMA_CQPSQ_FWQE_FLUSHSQ, flush_sq) |
+ FIELD_PREP(IRDMA_CQPSQ_FWQE_FLUSHRQ, flush_rq) |
+ FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
@@ -2417,12 +2410,14 @@
if (!wqe)
return -ENOSPC;
- temp = info->ae_code | LS_64(info->ae_src, IRDMA_CQPSQ_FWQE_AESOURCE);
+ temp = info->ae_code | FIELD_PREP(IRDMA_CQPSQ_FWQE_AESOURCE,
+ info->ae_src);
set_64bit_val(wqe, IRDMA_BYTE_8, temp);
- hdr = qp->qp_uk.qp_id | LS_64(IRDMA_CQP_OP_GEN_AE, IRDMA_CQPSQ_OPCODE) |
- LS_64(1, IRDMA_CQPSQ_FWQE_GENERATE_AE) |
- LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ hdr = qp->qp_uk.qp_id | FIELD_PREP(IRDMA_CQPSQ_OPCODE,
+ IRDMA_CQP_OP_GEN_AE) |
+ FIELD_PREP(IRDMA_CQPSQ_FWQE_GENERATE_AE, 1) |
+ FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
@@ -2457,12 +2452,12 @@
set_64bit_val(wqe, IRDMA_BYTE_16, info->buf_pa);
- hdr = LS_64(info->qp_id, IRDMA_CQPSQ_UCTX_QPID) |
- LS_64(IRDMA_CQP_OP_UPLOAD_CONTEXT, IRDMA_CQPSQ_OPCODE) |
- LS_64(info->qp_type, IRDMA_CQPSQ_UCTX_QPTYPE) |
- LS_64(info->raw_format, IRDMA_CQPSQ_UCTX_RAWFORMAT) |
- LS_64(info->freeze_qp, IRDMA_CQPSQ_UCTX_FREEZEQP) |
- LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ hdr = FIELD_PREP(IRDMA_CQPSQ_UCTX_QPID, info->qp_id) |
+ FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_UPLOAD_CONTEXT) |
+ FIELD_PREP(IRDMA_CQPSQ_UCTX_QPTYPE, info->qp_type) |
+ FIELD_PREP(IRDMA_CQPSQ_UCTX_RAWFORMAT, info->raw_format) |
+ FIELD_PREP(IRDMA_CQPSQ_UCTX_FREEZEQP, info->freeze_qp) |
+ FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
@@ -2499,11 +2494,11 @@
return -ENOSPC;
set_64bit_val(wqe, IRDMA_BYTE_16, info->qs_handle);
- hdr = LS_64(info->push_idx, IRDMA_CQPSQ_MPP_PPIDX) |
- LS_64(info->push_page_type, IRDMA_CQPSQ_MPP_PPTYPE) |
- LS_64(IRDMA_CQP_OP_MANAGE_PUSH_PAGES, IRDMA_CQPSQ_OPCODE) |
- LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID) |
- LS_64(info->free_page, IRDMA_CQPSQ_MPP_FREE_PAGE);
+ hdr = FIELD_PREP(IRDMA_CQPSQ_MPP_PPIDX, info->push_idx) |
+ FIELD_PREP(IRDMA_CQPSQ_MPP_PPTYPE, info->push_page_type) |
+ FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MANAGE_PUSH_PAGES) |
+ FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity) |
+ FIELD_PREP(IRDMA_CQPSQ_MPP_FREE_PAGE, info->free_page);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
@@ -2533,9 +2528,9 @@
if (!wqe)
return -ENOSPC;
- hdr = LS_64(qp->qp_uk.qp_id, IRDMA_CQPSQ_SUSPENDQP_QPID) |
- LS_64(IRDMA_CQP_OP_SUSPEND_QP, IRDMA_CQPSQ_OPCODE) |
- LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ hdr = FIELD_PREP(IRDMA_CQPSQ_SUSPENDQP_QPID, qp->qp_uk.qp_id) |
+ FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_SUSPEND_QP) |
+ FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
@@ -2565,11 +2560,11 @@
return -ENOSPC;
set_64bit_val(wqe, IRDMA_BYTE_16,
- LS_64(qp->qs_handle, IRDMA_CQPSQ_RESUMEQP_QSHANDLE));
+ FIELD_PREP(IRDMA_CQPSQ_RESUMEQP_QSHANDLE, qp->qs_handle));
- hdr = LS_64(qp->qp_uk.qp_id, IRDMA_CQPSQ_RESUMEQP_QPID) |
- LS_64(IRDMA_CQP_OP_RESUME_QP, IRDMA_CQPSQ_OPCODE) |
- LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ hdr = FIELD_PREP(IRDMA_CQPSQ_RESUMEQP_QPID, qp->qp_uk.qp_id) |
+ FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_RESUME_QP) |
+ FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
@@ -2671,29 +2666,29 @@
set_64bit_val(wqe, IRDMA_BYTE_0, cq->cq_uk.cq_size);
set_64bit_val(wqe, IRDMA_BYTE_8, RS_64_1(cq, 1));
set_64bit_val(wqe, IRDMA_BYTE_16,
- LS_64(cq->shadow_read_threshold,
- IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD));
+ FIELD_PREP(IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD, cq->shadow_read_threshold));
set_64bit_val(wqe, IRDMA_BYTE_32, (cq->virtual_map ? 0 : cq->cq_pa));
set_64bit_val(wqe, IRDMA_BYTE_40, cq->shadow_area_pa);
set_64bit_val(wqe, IRDMA_BYTE_48,
- LS_64((cq->virtual_map ? cq->first_pm_pbl_idx : 0),
- IRDMA_CQPSQ_CQ_FIRSTPMPBLIDX));
+ FIELD_PREP(IRDMA_CQPSQ_CQ_FIRSTPMPBLIDX, (cq->virtual_map ? cq->first_pm_pbl_idx : 0)));
set_64bit_val(wqe, IRDMA_BYTE_56,
- LS_64(cq->tph_val, IRDMA_CQPSQ_TPHVAL) |
- LS_64(cq->vsi->vsi_idx, IRDMA_CQPSQ_VSIIDX));
+ FIELD_PREP(IRDMA_CQPSQ_TPHVAL, cq->tph_val) |
+ FIELD_PREP(IRDMA_CQPSQ_VSIIDX, cq->vsi->vsi_idx));
hdr = FLD_LS_64(cq->dev, cq->cq_uk.cq_id, IRDMA_CQPSQ_CQ_CQID) |
FLD_LS_64(cq->dev, (cq->ceq_id_valid ? cq->ceq_id : 0),
IRDMA_CQPSQ_CQ_CEQID) |
- LS_64(IRDMA_CQP_OP_CREATE_CQ, IRDMA_CQPSQ_OPCODE) |
- LS_64(cq->pbl_chunk_size, IRDMA_CQPSQ_CQ_LPBLSIZE) |
- LS_64(check_overflow, IRDMA_CQPSQ_CQ_CHKOVERFLOW) |
- LS_64(cq->virtual_map, IRDMA_CQPSQ_CQ_VIRTMAP) |
- LS_64(cq->ceqe_mask, IRDMA_CQPSQ_CQ_ENCEQEMASK) |
- LS_64(cq->ceq_id_valid, IRDMA_CQPSQ_CQ_CEQIDVALID) |
- LS_64(cq->tph_en, IRDMA_CQPSQ_TPHEN) |
- LS_64(cq->cq_uk.avoid_mem_cflct, IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT) |
- LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_CQ) |
+ FIELD_PREP(IRDMA_CQPSQ_CQ_LPBLSIZE, cq->pbl_chunk_size) |
+ FIELD_PREP(IRDMA_CQPSQ_CQ_CHKOVERFLOW, check_overflow) |
+ FIELD_PREP(IRDMA_CQPSQ_CQ_VIRTMAP, cq->virtual_map) |
+ FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, cq->ceqe_mask) |
+ FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, cq->ceq_id_valid) |
+ FIELD_PREP(IRDMA_CQPSQ_TPHEN, cq->tph_en) |
+ FIELD_PREP(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT,
+ cq->cq_uk.avoid_mem_cflct) |
+ FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
+
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
@@ -2738,14 +2733,14 @@
hdr = cq->cq_uk.cq_id |
FLD_LS_64(cq->dev, (cq->ceq_id_valid ? cq->ceq_id : 0),
IRDMA_CQPSQ_CQ_CEQID) |
- LS_64(IRDMA_CQP_OP_DESTROY_CQ, IRDMA_CQPSQ_OPCODE) |
- LS_64(cq->pbl_chunk_size, IRDMA_CQPSQ_CQ_LPBLSIZE) |
- LS_64(cq->virtual_map, IRDMA_CQPSQ_CQ_VIRTMAP) |
- LS_64(cq->ceqe_mask, IRDMA_CQPSQ_CQ_ENCEQEMASK) |
- LS_64(cq->ceq_id_valid, IRDMA_CQPSQ_CQ_CEQIDVALID) |
- LS_64(cq->tph_en, IRDMA_CQPSQ_TPHEN) |
- LS_64(cq->cq_uk.avoid_mem_cflct, IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT) |
- LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_CQ) |
+ FIELD_PREP(IRDMA_CQPSQ_CQ_LPBLSIZE, cq->pbl_chunk_size) |
+ FIELD_PREP(IRDMA_CQPSQ_CQ_VIRTMAP, cq->virtual_map) |
+ FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, cq->ceqe_mask) |
+ FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, cq->ceq_id_valid) |
+ FIELD_PREP(IRDMA_CQPSQ_TPHEN, cq->tph_en) |
+ FIELD_PREP(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT, cq->cq_uk.avoid_mem_cflct) |
+ FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
@@ -2803,25 +2798,25 @@
set_64bit_val(wqe, IRDMA_BYTE_0, info->cq_size);
set_64bit_val(wqe, IRDMA_BYTE_8, RS_64_1(cq, 1));
set_64bit_val(wqe, IRDMA_BYTE_16,
- LS_64(info->shadow_read_threshold,
- IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD));
+ FIELD_PREP(IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD, info->shadow_read_threshold));
set_64bit_val(wqe, IRDMA_BYTE_32, info->cq_pa);
set_64bit_val(wqe, IRDMA_BYTE_40, cq->shadow_area_pa);
set_64bit_val(wqe, IRDMA_BYTE_48, info->first_pm_pbl_idx);
set_64bit_val(wqe, IRDMA_BYTE_56,
- LS_64(cq->tph_val, IRDMA_CQPSQ_TPHVAL) |
- LS_64(cq->vsi->vsi_idx, IRDMA_CQPSQ_VSIIDX));
+ FIELD_PREP(IRDMA_CQPSQ_TPHVAL, cq->tph_val) |
+ FIELD_PREP(IRDMA_CQPSQ_VSIIDX, cq->vsi->vsi_idx));
hdr = cq->cq_uk.cq_id |
- LS_64(IRDMA_CQP_OP_MODIFY_CQ, IRDMA_CQPSQ_OPCODE) |
- LS_64(info->cq_resize, IRDMA_CQPSQ_CQ_CQRESIZE) |
- LS_64(info->pbl_chunk_size, IRDMA_CQPSQ_CQ_LPBLSIZE) |
- LS_64(info->check_overflow, IRDMA_CQPSQ_CQ_CHKOVERFLOW) |
- LS_64(info->virtual_map, IRDMA_CQPSQ_CQ_VIRTMAP) |
- LS_64(cq->ceqe_mask, IRDMA_CQPSQ_CQ_ENCEQEMASK) |
- LS_64(cq->tph_en, IRDMA_CQPSQ_TPHEN) |
- LS_64(cq->cq_uk.avoid_mem_cflct, IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT) |
- LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MODIFY_CQ) |
+ FIELD_PREP(IRDMA_CQPSQ_CQ_CQRESIZE, info->cq_resize) |
+ FIELD_PREP(IRDMA_CQPSQ_CQ_LPBLSIZE, info->pbl_chunk_size) |
+ FIELD_PREP(IRDMA_CQPSQ_CQ_CHKOVERFLOW, info->check_overflow) |
+ FIELD_PREP(IRDMA_CQPSQ_CQ_VIRTMAP, info->virtual_map) |
+ FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, cq->ceqe_mask) |
+ FIELD_PREP(IRDMA_CQPSQ_TPHEN, cq->tph_en) |
+ FIELD_PREP(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT,
+ cq->cq_uk.avoid_mem_cflct) |
+ FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
@@ -2840,15 +2835,15 @@
* @dev: sc device struct
*/
void
-irdma_check_cqp_progress(struct irdma_cqp_timeout *timeout, struct irdma_sc_dev *dev)
+irdma_check_cqp_progress(struct irdma_cqp_timeout *timeout,
+ struct irdma_sc_dev *dev)
{
if (timeout->compl_cqp_cmds != dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS]) {
timeout->compl_cqp_cmds = dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS];
timeout->count = 0;
- } else {
- if (dev->cqp_cmd_stats[IRDMA_OP_REQ_CMDS] !=
- timeout->compl_cqp_cmds)
- timeout->count++;
+ } else if (timeout->compl_cqp_cmds !=
+ dev->cqp_cmd_stats[IRDMA_OP_REQ_CMDS]) {
+ timeout->count++;
}
}
@@ -2864,8 +2859,8 @@
u32 *tail, u32 *error)
{
*val = readl(cqp->dev->hw_regs[IRDMA_CQPTAIL]);
- *tail = RS_32(*val, IRDMA_CQPTAIL_WQTAIL);
- *error = RS_32(*val, IRDMA_CQPTAIL_CQP_OP_ERR);
+ *tail = FIELD_GET(IRDMA_CQPTAIL_WQTAIL, *val);
+ *error = FIELD_GET(IRDMA_CQPTAIL_CQP_OP_ERR, *val);
}
/**
@@ -2918,13 +2913,16 @@
switch (rsrc_idx) {
case IRDMA_HMC_IW_QP:
- obj_info[rsrc_idx].cnt = (u32)RS_64(temp, IRDMA_COMMIT_FPM_QPCNT);
+ obj_info[rsrc_idx].cnt = (u32)FIELD_GET(IRDMA_COMMIT_FPM_QPCNT, temp);
break;
case IRDMA_HMC_IW_CQ:
obj_info[rsrc_idx].cnt = (u32)FLD_RS_64(dev, temp, IRDMA_COMMIT_FPM_CQCNT);
break;
case IRDMA_HMC_IW_APBVT_ENTRY:
- obj_info[rsrc_idx].cnt = 1;
+ if (dev->hw_attrs.uk_attrs.hw_rev <= IRDMA_GEN_2)
+ obj_info[rsrc_idx].cnt = 1;
+ else
+ obj_info[rsrc_idx].cnt = 0;
break;
default:
obj_info[rsrc_idx].cnt = (u32)temp;
@@ -3066,18 +3064,18 @@
obj_info = hmc_info->hmc_obj;
get_64bit_val(buf, IRDMA_BYTE_0, &temp);
- hmc_info->first_sd_index = (u16)RS_64(temp, IRDMA_QUERY_FPM_FIRST_PE_SD_INDEX);
- max_pe_sds = (u16)RS_64(temp, IRDMA_QUERY_FPM_MAX_PE_SDS);
+ hmc_info->first_sd_index = (u16)FIELD_GET(IRDMA_QUERY_FPM_FIRST_PE_SD_INDEX, temp);
+ max_pe_sds = (u16)FIELD_GET(IRDMA_QUERY_FPM_MAX_PE_SDS, temp);
hmc_fpm_misc->max_sds = max_pe_sds;
hmc_info->sd_table.sd_cnt = max_pe_sds + hmc_info->first_sd_index;
get_64bit_val(buf, 8, &temp);
- obj_info[IRDMA_HMC_IW_QP].max_cnt = (u32)RS_64(temp, IRDMA_QUERY_FPM_MAX_QPS);
+ obj_info[IRDMA_HMC_IW_QP].max_cnt = (u32)FIELD_GET(IRDMA_QUERY_FPM_MAX_QPS, temp);
size = (u32)RS_64_1(temp, 32);
obj_info[IRDMA_HMC_IW_QP].size = LS_64_1(1, size);
get_64bit_val(buf, 16, &temp);
- obj_info[IRDMA_HMC_IW_CQ].max_cnt = (u32)RS_64(temp, IRDMA_QUERY_FPM_MAX_CQS);
+ obj_info[IRDMA_HMC_IW_CQ].max_cnt = (u32)FIELD_GET(IRDMA_QUERY_FPM_MAX_CQS, temp);
size = (u32)RS_64_1(temp, 32);
obj_info[IRDMA_HMC_IW_CQ].size = LS_64_1(1, size);
@@ -3093,7 +3091,7 @@
get_64bit_val(buf, 64, &temp);
obj_info[IRDMA_HMC_IW_XFFL].max_cnt = (u32)temp;
obj_info[IRDMA_HMC_IW_XFFL].size = 4;
- hmc_fpm_misc->xf_block_size = RS_64(temp, IRDMA_QUERY_FPM_XFBLOCKSIZE);
+ hmc_fpm_misc->xf_block_size = FIELD_GET(IRDMA_QUERY_FPM_XFBLOCKSIZE, temp);
if (!hmc_fpm_misc->xf_block_size)
return -EINVAL;
@@ -3102,7 +3100,7 @@
obj_info[IRDMA_HMC_IW_Q1FL].max_cnt = (u32)temp;
obj_info[IRDMA_HMC_IW_Q1FL].size = 4;
- hmc_fpm_misc->q1_block_size = RS_64(temp, IRDMA_QUERY_FPM_Q1BLOCKSIZE);
+ hmc_fpm_misc->q1_block_size = FIELD_GET(IRDMA_QUERY_FPM_Q1BLOCKSIZE, temp);
if (!hmc_fpm_misc->q1_block_size)
return -EINVAL;
@@ -3113,9 +3111,9 @@
obj_info[IRDMA_HMC_IW_PBLE].size = 8;
get_64bit_val(buf, 120, &temp);
- hmc_fpm_misc->max_ceqs = RS_64(temp, IRDMA_QUERY_FPM_MAX_CEQS);
- hmc_fpm_misc->ht_multiplier = RS_64(temp, IRDMA_QUERY_FPM_HTMULTIPLIER);
- hmc_fpm_misc->timer_bucket = RS_64(temp, IRDMA_QUERY_FPM_TIMERBUCKET);
+ hmc_fpm_misc->max_ceqs = FIELD_GET(IRDMA_QUERY_FPM_MAX_CEQS, temp);
+ hmc_fpm_misc->ht_multiplier = FIELD_GET(IRDMA_QUERY_FPM_HTMULTIPLIER, temp);
+ hmc_fpm_misc->timer_bucket = FIELD_GET(IRDMA_QUERY_FPM_TIMERBUCKET, temp);
if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
return 0;
irdma_sc_decode_fpm_query(buf, 96, obj_info, IRDMA_HMC_IW_FSIMC);
@@ -3125,7 +3123,7 @@
get_64bit_val(buf, IRDMA_BYTE_136, &temp);
obj_info[IRDMA_HMC_IW_RRFFL].max_cnt = (u32)temp;
obj_info[IRDMA_HMC_IW_RRFFL].size = 4;
- hmc_fpm_misc->rrf_block_size = RS_64(temp, IRDMA_QUERY_FPM_RRFBLOCKSIZE);
+ hmc_fpm_misc->rrf_block_size = FIELD_GET(IRDMA_QUERY_FPM_RRFBLOCKSIZE, temp);
if (!hmc_fpm_misc->rrf_block_size &&
obj_info[IRDMA_HMC_IW_RRFFL].max_cnt)
return -EINVAL;
@@ -3137,7 +3135,7 @@
get_64bit_val(buf, IRDMA_BYTE_168, &temp);
obj_info[IRDMA_HMC_IW_OOISCFFL].max_cnt = (u32)temp;
obj_info[IRDMA_HMC_IW_OOISCFFL].size = 4;
- hmc_fpm_misc->ooiscf_block_size = RS_64(temp, IRDMA_QUERY_FPM_OOISCFBLOCKSIZE);
+ hmc_fpm_misc->ooiscf_block_size = FIELD_GET(IRDMA_QUERY_FPM_OOISCFBLOCKSIZE, temp);
if (!hmc_fpm_misc->ooiscf_block_size &&
obj_info[IRDMA_HMC_IW_OOISCFFL].max_cnt)
return -EINVAL;
@@ -3296,45 +3294,48 @@
spin_lock_init(&cqp->dev->cqp_lock);
- temp = LS_64(cqp->hw_sq_size, IRDMA_CQPHC_SQSIZE) |
- LS_64(cqp->struct_ver, IRDMA_CQPHC_SVER) |
- LS_64(cqp->disable_packed, IRDMA_CQPHC_DISABLE_PFPDUS) |
- LS_64(cqp->ceqs_per_vf, IRDMA_CQPHC_CEQPERVF);
+ temp = FIELD_PREP(IRDMA_CQPHC_SQSIZE, cqp->hw_sq_size) |
+ FIELD_PREP(IRDMA_CQPHC_SVER, cqp->struct_ver) |
+ FIELD_PREP(IRDMA_CQPHC_DISABLE_PFPDUS, cqp->disable_packed) |
+ FIELD_PREP(IRDMA_CQPHC_CEQPERVF, cqp->ceqs_per_vf);
if (hw_rev >= IRDMA_GEN_2) {
- temp |= LS_64(cqp->rocev2_rto_policy, IRDMA_CQPHC_ROCEV2_RTO_POLICY) |
- LS_64(cqp->protocol_used, IRDMA_CQPHC_PROTOCOL_USED);
+ temp |= FIELD_PREP(IRDMA_CQPHC_ROCEV2_RTO_POLICY,
+ cqp->rocev2_rto_policy) |
+ FIELD_PREP(IRDMA_CQPHC_PROTOCOL_USED,
+ cqp->protocol_used);
}
set_64bit_val(cqp->host_ctx, IRDMA_BYTE_0, temp);
set_64bit_val(cqp->host_ctx, IRDMA_BYTE_8, cqp->sq_pa);
- temp = LS_64(cqp->ena_vf_count, IRDMA_CQPHC_ENABLED_VFS) |
- LS_64(cqp->hmc_profile, IRDMA_CQPHC_HMC_PROFILE);
+ temp = FIELD_PREP(IRDMA_CQPHC_ENABLED_VFS, cqp->ena_vf_count) |
+ FIELD_PREP(IRDMA_CQPHC_HMC_PROFILE, cqp->hmc_profile);
if (hw_rev >= IRDMA_GEN_2)
- temp |= LS_64(cqp->en_rem_endpoint_trk, IRDMA_CQPHC_EN_REM_ENDPOINT_TRK);
+ temp |= FIELD_PREP(IRDMA_CQPHC_EN_REM_ENDPOINT_TRK,
+ cqp->en_rem_endpoint_trk);
set_64bit_val(cqp->host_ctx, IRDMA_BYTE_16, temp);
set_64bit_val(cqp->host_ctx, IRDMA_BYTE_24, (uintptr_t)cqp);
- temp = LS_64(cqp->hw_maj_ver, IRDMA_CQPHC_HW_MAJVER) |
- LS_64(cqp->hw_min_ver, IRDMA_CQPHC_HW_MINVER);
+ temp = FIELD_PREP(IRDMA_CQPHC_HW_MAJVER, cqp->hw_maj_ver) |
+ FIELD_PREP(IRDMA_CQPHC_HW_MINVER, cqp->hw_min_ver);
if (hw_rev >= IRDMA_GEN_2) {
- temp |= LS_64(cqp->dcqcn_params.min_rate, IRDMA_CQPHC_MIN_RATE) |
- LS_64(cqp->dcqcn_params.min_dec_factor, IRDMA_CQPHC_MIN_DEC_FACTOR);
+ temp |= FIELD_PREP(IRDMA_CQPHC_MIN_RATE, cqp->dcqcn_params.min_rate) |
+ FIELD_PREP(IRDMA_CQPHC_MIN_DEC_FACTOR, cqp->dcqcn_params.min_dec_factor);
}
set_64bit_val(cqp->host_ctx, IRDMA_BYTE_32, temp);
set_64bit_val(cqp->host_ctx, IRDMA_BYTE_40, 0);
temp = 0;
if (hw_rev >= IRDMA_GEN_2) {
- temp |= LS_64(cqp->dcqcn_params.dcqcn_t, IRDMA_CQPHC_DCQCN_T) |
- LS_64(cqp->dcqcn_params.rai_factor, IRDMA_CQPHC_RAI_FACTOR) |
- LS_64(cqp->dcqcn_params.hai_factor, IRDMA_CQPHC_HAI_FACTOR);
+ temp |= FIELD_PREP(IRDMA_CQPHC_DCQCN_T, cqp->dcqcn_params.dcqcn_t) |
+ FIELD_PREP(IRDMA_CQPHC_RAI_FACTOR, cqp->dcqcn_params.rai_factor) |
+ FIELD_PREP(IRDMA_CQPHC_HAI_FACTOR, cqp->dcqcn_params.hai_factor);
}
set_64bit_val(cqp->host_ctx, IRDMA_BYTE_48, temp);
temp = 0;
if (hw_rev >= IRDMA_GEN_2) {
- temp |= LS_64(cqp->dcqcn_params.dcqcn_b, IRDMA_CQPHC_DCQCN_B) |
- LS_64(cqp->dcqcn_params.dcqcn_f, IRDMA_CQPHC_DCQCN_F) |
- LS_64(cqp->dcqcn_params.cc_cfg_valid, IRDMA_CQPHC_CC_CFG_VALID) |
- LS_64(cqp->dcqcn_params.rreduce_mperiod, IRDMA_CQPHC_RREDUCE_MPERIOD);
+ temp |= FIELD_PREP(IRDMA_CQPHC_DCQCN_B, cqp->dcqcn_params.dcqcn_b) |
+ FIELD_PREP(IRDMA_CQPHC_DCQCN_F, cqp->dcqcn_params.dcqcn_f) |
+ FIELD_PREP(IRDMA_CQPHC_CC_CFG_VALID, cqp->dcqcn_params.cc_cfg_valid) |
+ FIELD_PREP(IRDMA_CQPHC_RREDUCE_MPERIOD, cqp->dcqcn_params.rreduce_mperiod);
}
set_64bit_val(cqp->host_ctx, IRDMA_BYTE_56, temp);
irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "CQP_HOST_CTX WQE",
@@ -3366,8 +3367,8 @@
spin_lock_destroy(&cqp->dev->cqp_lock);
irdma_free_dma_mem(cqp->dev->hw, &cqp->sdbuf);
err_code = readl(cqp->dev->hw_regs[IRDMA_CQPERRCODES]);
- *min_err = RS_32(err_code, IRDMA_CQPERRCODES_CQP_MINOR_CODE);
- *maj_err = RS_32(err_code, IRDMA_CQPERRCODES_CQP_MAJOR_CODE);
+ *min_err = FIELD_GET(IRDMA_CQPERRCODES_CQP_MINOR_CODE, err_code);
+ *maj_err = FIELD_GET(IRDMA_CQPERRCODES_CQP_MAJOR_CODE, err_code);
return ret_code;
}
@@ -3425,24 +3426,26 @@
/**
* irdma_sc_cqp_destroy - destroy cqp during close
* @cqp: struct for cqp hw
+ * @free_hwcqp: true for regular cqp destroy; false for reset path
*/
int
-irdma_sc_cqp_destroy(struct irdma_sc_cqp *cqp)
+irdma_sc_cqp_destroy(struct irdma_sc_cqp *cqp, bool free_hwcqp)
{
u32 cnt = 0, val;
int ret_code = 0;
- writel(0, cqp->dev->hw_regs[IRDMA_CCQPHIGH]);
- writel(0, cqp->dev->hw_regs[IRDMA_CCQPLOW]);
- do {
- if (cnt++ > cqp->dev->hw_attrs.max_done_count) {
- ret_code = -ETIMEDOUT;
- break;
- }
- irdma_usec_delay(cqp->dev->hw_attrs.max_sleep_count);
- val = readl(cqp->dev->hw_regs[IRDMA_CCQPSTATUS]);
- } while (FLD_RS_32(cqp->dev, val, IRDMA_CCQPSTATUS_CCQP_DONE));
-
+ if (free_hwcqp) {
+ writel(0, cqp->dev->hw_regs[IRDMA_CCQPHIGH]);
+ writel(0, cqp->dev->hw_regs[IRDMA_CCQPLOW]);
+ do {
+ if (cnt++ > cqp->dev->hw_attrs.max_done_count) {
+ ret_code = -ETIMEDOUT;
+ break;
+ }
+ irdma_usec_delay(cqp->dev->hw_attrs.max_sleep_count);
+ val = readl(cqp->dev->hw_regs[IRDMA_CCQPSTATUS]);
+ } while (FLD_RS_32(cqp->dev, val, IRDMA_CCQPSTATUS_CCQP_DONE));
+ }
irdma_free_dma_mem(cqp->dev->hw, &cqp->sdbuf);
spin_lock_destroy(&cqp->dev->cqp_lock);
return ret_code;
@@ -3461,14 +3464,14 @@
u8 arm_seq_num;
get_64bit_val(ccq->cq_uk.shadow_area, IRDMA_BYTE_32, &temp_val);
- sw_cq_sel = (u16)RS_64(temp_val, IRDMA_CQ_DBSA_SW_CQ_SELECT);
- arm_next_se = (u8)RS_64(temp_val, IRDMA_CQ_DBSA_ARM_NEXT_SE);
- arm_seq_num = (u8)RS_64(temp_val, IRDMA_CQ_DBSA_ARM_SEQ_NUM);
+ sw_cq_sel = (u16)FIELD_GET(IRDMA_CQ_DBSA_SW_CQ_SELECT, temp_val);
+ arm_next_se = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT_SE, temp_val);
+ arm_seq_num = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_SEQ_NUM, temp_val);
arm_seq_num++;
- temp_val = LS_64(arm_seq_num, IRDMA_CQ_DBSA_ARM_SEQ_NUM) |
- LS_64(sw_cq_sel, IRDMA_CQ_DBSA_SW_CQ_SELECT) |
- LS_64(arm_next_se, IRDMA_CQ_DBSA_ARM_NEXT_SE) |
- LS_64(1, IRDMA_CQ_DBSA_ARM_NEXT);
+ temp_val = FIELD_PREP(IRDMA_CQ_DBSA_ARM_SEQ_NUM, arm_seq_num) |
+ FIELD_PREP(IRDMA_CQ_DBSA_SW_CQ_SELECT, sw_cq_sel) |
+ FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT_SE, arm_next_se) |
+ FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT, 1);
set_64bit_val(ccq->cq_uk.shadow_area, IRDMA_BYTE_32, temp_val);
irdma_wmb(); /* make sure shadow area is updated before arming */
@@ -3499,29 +3502,30 @@
cqe = IRDMA_GET_CURRENT_CQ_ELEM(&ccq->cq_uk);
get_64bit_val(cqe, IRDMA_BYTE_24, &temp);
- polarity = (u8)RS_64(temp, IRDMA_CQ_VALID);
+ polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, temp);
if (polarity != ccq->cq_uk.polarity)
return -ENOENT;
get_64bit_val(cqe, IRDMA_BYTE_8, &qp_ctx);
cqp = (struct irdma_sc_cqp *)(irdma_uintptr) qp_ctx;
- info->error = (bool)RS_64(temp, IRDMA_CQ_ERROR);
+ info->error = (bool)FIELD_GET(IRDMA_CQ_ERROR, temp);
info->maj_err_code = IRDMA_CQPSQ_MAJ_NO_ERROR;
- info->min_err_code = (u16)RS_64(temp, IRDMA_CQ_MINERR);
+ info->min_err_code = (u16)FIELD_GET(IRDMA_CQ_MINERR, temp);
if (info->error) {
- info->maj_err_code = (u16)RS_64(temp, IRDMA_CQ_MAJERR);
+ info->maj_err_code = (u16)FIELD_GET(IRDMA_CQ_MAJERR, temp);
error = readl(cqp->dev->hw_regs[IRDMA_CQPERRCODES]);
irdma_debug(cqp->dev, IRDMA_DEBUG_CQP,
"CQPERRCODES error_code[x%08X]\n", error);
}
- wqe_idx = (u32)RS_64(temp, IRDMA_CQ_WQEIDX);
+ wqe_idx = (u32)FIELD_GET(IRDMA_CQ_WQEIDX, temp);
info->scratch = cqp->scratch_array[wqe_idx];
get_64bit_val(cqe, IRDMA_BYTE_16, &temp1);
- info->op_ret_val = (u32)RS_64(temp1, IRDMA_CCQ_OPRETVAL);
+ info->op_ret_val = (u32)FIELD_GET(IRDMA_CCQ_OPRETVAL, temp1);
+
get_64bit_val(cqp->sq_base[wqe_idx].elem, IRDMA_BYTE_24, &temp1);
- info->op_code = (u8)RS_64(temp1, IRDMA_CQPSQ_OPCODE);
+ info->op_code = (u8)FIELD_GET(IRDMA_CQPSQ_OPCODE, temp1);
info->cqp = cqp;
/* move the head for cq */
@@ -3562,9 +3566,6 @@
if (cnt++ > 100 * cqp->dev->hw_attrs.max_done_count)
return -ETIMEDOUT;
- if (cqp->dev->no_cqp)
- return -ETIMEDOUT;
-
if (irdma_sc_ccq_get_cqe_info(ccq, &info)) {
irdma_usec_delay(cqp->dev->hw_attrs.max_sleep_count);
continue;
@@ -3606,11 +3607,11 @@
if (!wqe)
return -ENOSPC;
- hdr = LS_64(info->vf_id, IRDMA_CQPSQ_MHMC_VFIDX) |
- LS_64(IRDMA_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE,
- IRDMA_CQPSQ_OPCODE) |
- LS_64(info->free_fcn, IRDMA_CQPSQ_MHMC_FREEPMFN) |
- LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ hdr = FIELD_PREP(IRDMA_CQPSQ_MHMC_VFIDX, info->vf_id) |
+ FIELD_PREP(IRDMA_CQPSQ_OPCODE,
+ IRDMA_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE) |
+ FIELD_PREP(IRDMA_CQPSQ_MHMC_FREEPMFN, info->free_fcn) |
+ FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
@@ -3647,7 +3648,7 @@
*/
static int
irdma_sc_commit_fpm_val(struct irdma_sc_cqp *cqp, u64 scratch,
- u8 hmc_fn_id,
+ u16 hmc_fn_id,
struct irdma_dma_mem *commit_fpm_mem,
bool post_sq, u8 wait_type)
{
@@ -3663,9 +3664,9 @@
set_64bit_val(wqe, IRDMA_BYTE_16, hmc_fn_id);
set_64bit_val(wqe, IRDMA_BYTE_32, commit_fpm_mem->pa);
- hdr = LS_64(IRDMA_COMMIT_FPM_BUF_SIZE, IRDMA_CQPSQ_BUFSIZE) |
- LS_64(IRDMA_CQP_OP_COMMIT_FPM_VAL, IRDMA_CQPSQ_OPCODE) |
- LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ hdr = FIELD_PREP(IRDMA_CQPSQ_BUFSIZE, IRDMA_COMMIT_FPM_BUF_SIZE) |
+ FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_COMMIT_FPM_VAL) |
+ FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
@@ -3710,7 +3711,7 @@
*/
static int
irdma_sc_query_fpm_val(struct irdma_sc_cqp *cqp, u64 scratch,
- u8 hmc_fn_id,
+ u16 hmc_fn_id,
struct irdma_dma_mem *query_fpm_mem,
bool post_sq, u8 wait_type)
{
@@ -3726,8 +3727,8 @@
set_64bit_val(wqe, IRDMA_BYTE_16, hmc_fn_id);
set_64bit_val(wqe, IRDMA_BYTE_32, query_fpm_mem->pa);
- hdr = LS_64(IRDMA_CQP_OP_QUERY_FPM_VAL, IRDMA_CQPSQ_OPCODE) |
- LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_QUERY_FPM_VAL) |
+ FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
@@ -3800,7 +3801,6 @@
* @scratch: u64 saved to be used during cqp completion
* @post_sq: flag for cqp db to ring
*/
-
static int
irdma_sc_ceq_create(struct irdma_sc_ceq *ceq, u64 scratch,
bool post_sq)
@@ -3819,15 +3819,15 @@
set_64bit_val(wqe, IRDMA_BYTE_48,
(ceq->virtual_map ? ceq->first_pm_pbl_idx : 0));
set_64bit_val(wqe, IRDMA_BYTE_56,
- LS_64(ceq->tph_val, IRDMA_CQPSQ_TPHVAL) |
- LS_64(ceq->vsi->vsi_idx, IRDMA_CQPSQ_VSIIDX));
- hdr = LS_64(ceq->ceq_id, IRDMA_CQPSQ_CEQ_CEQID) |
- LS_64(IRDMA_CQP_OP_CREATE_CEQ, IRDMA_CQPSQ_OPCODE) |
- LS_64(ceq->pbl_chunk_size, IRDMA_CQPSQ_CEQ_LPBLSIZE) |
- LS_64(ceq->virtual_map, IRDMA_CQPSQ_CEQ_VMAP) |
- LS_64(ceq->itr_no_expire, IRDMA_CQPSQ_CEQ_ITRNOEXPIRE) |
- LS_64(ceq->tph_en, IRDMA_CQPSQ_TPHEN) |
- LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ FIELD_PREP(IRDMA_CQPSQ_TPHVAL, ceq->tph_val) |
+ FIELD_PREP(IRDMA_CQPSQ_VSIIDX, ceq->vsi->vsi_idx));
+ hdr = FIELD_PREP(IRDMA_CQPSQ_CEQ_CEQID, ceq->ceq_id) |
+ FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_CEQ) |
+ FIELD_PREP(IRDMA_CQPSQ_CEQ_LPBLSIZE, ceq->pbl_chunk_size) |
+ FIELD_PREP(IRDMA_CQPSQ_CEQ_VMAP, ceq->virtual_map) |
+ FIELD_PREP(IRDMA_CQPSQ_CEQ_ITRNOEXPIRE, ceq->itr_no_expire) |
+ FIELD_PREP(IRDMA_CQPSQ_TPHEN, ceq->tph_en) |
+ FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
@@ -3919,11 +3919,11 @@
set_64bit_val(wqe, IRDMA_BYTE_16, ceq->elem_cnt);
set_64bit_val(wqe, IRDMA_BYTE_48, ceq->first_pm_pbl_idx);
hdr = ceq->ceq_id |
- LS_64(IRDMA_CQP_OP_DESTROY_CEQ, IRDMA_CQPSQ_OPCODE) |
- LS_64(ceq->pbl_chunk_size, IRDMA_CQPSQ_CEQ_LPBLSIZE) |
- LS_64(ceq->virtual_map, IRDMA_CQPSQ_CEQ_VMAP) |
- LS_64(ceq->tph_en, IRDMA_CQPSQ_TPHEN) |
- LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_CEQ) |
+ FIELD_PREP(IRDMA_CQPSQ_CEQ_LPBLSIZE, ceq->pbl_chunk_size) |
+ FIELD_PREP(IRDMA_CQPSQ_CEQ_VMAP, ceq->virtual_map) |
+ FIELD_PREP(IRDMA_CQPSQ_TPHEN, ceq->tph_en) |
+ FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
@@ -3960,7 +3960,7 @@
cq_idx = 0;
ceqe = IRDMA_GET_CURRENT_CEQ_ELEM(ceq);
get_64bit_val(ceqe, IRDMA_BYTE_0, &temp);
- polarity = (u8)RS_64(temp, IRDMA_CEQE_VALID);
+ polarity = (u8)FIELD_GET(IRDMA_CEQE_VALID, temp);
if (polarity != ceq->polarity)
return NULL;
@@ -4019,13 +4019,13 @@
ceqe = IRDMA_GET_CEQ_ELEM_AT_POS(ceq, next);
get_64bit_val(ceqe, IRDMA_BYTE_0, &temp);
- polarity = (u8)RS_64(temp, IRDMA_CEQE_VALID);
+ polarity = (u8)FIELD_GET(IRDMA_CEQE_VALID, temp);
if (polarity != ceq_polarity)
return;
next_cq = (struct irdma_sc_cq *)(irdma_uintptr) LS_64_1(temp, 1);
if (cq == next_cq)
- set_64bit_val(ceqe, IRDMA_BYTE_0, temp & IRDMA_CEQE_VALID_M);
+ set_64bit_val(ceqe, IRDMA_BYTE_0, temp & IRDMA_CEQE_VALID);
next = IRDMA_RING_GET_NEXT_TAIL(ceq->ceq_ring, i);
if (!next)
@@ -4094,10 +4094,10 @@
set_64bit_val(wqe, IRDMA_BYTE_48,
(aeq->virtual_map ? aeq->first_pm_pbl_idx : 0));
- hdr = LS_64(IRDMA_CQP_OP_CREATE_AEQ, IRDMA_CQPSQ_OPCODE) |
- LS_64(aeq->pbl_chunk_size, IRDMA_CQPSQ_AEQ_LPBLSIZE) |
- LS_64(aeq->virtual_map, IRDMA_CQPSQ_AEQ_VMAP) |
- LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_AEQ) |
+ FIELD_PREP(IRDMA_CQPSQ_AEQ_LPBLSIZE, aeq->pbl_chunk_size) |
+ FIELD_PREP(IRDMA_CQPSQ_AEQ_VMAP, aeq->virtual_map) |
+ FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
@@ -4116,9 +4116,8 @@
* @scratch: u64 saved to be used during cqp completion
* @post_sq: flag for cqp db to ring
*/
-static int
-irdma_sc_aeq_destroy(struct irdma_sc_aeq *aeq, u64 scratch,
- bool post_sq)
+int
+irdma_sc_aeq_destroy(struct irdma_sc_aeq *aeq, u64 scratch, bool post_sq)
{
__le64 *wqe;
struct irdma_sc_cqp *cqp;
@@ -4134,10 +4133,10 @@
return -ENOSPC;
set_64bit_val(wqe, IRDMA_BYTE_16, aeq->elem_cnt);
set_64bit_val(wqe, IRDMA_BYTE_48, aeq->first_pm_pbl_idx);
- hdr = LS_64(IRDMA_CQP_OP_DESTROY_AEQ, IRDMA_CQPSQ_OPCODE) |
- LS_64(aeq->pbl_chunk_size, IRDMA_CQPSQ_AEQ_LPBLSIZE) |
- LS_64(aeq->virtual_map, IRDMA_CQPSQ_AEQ_VMAP) |
- LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_AEQ) |
+ FIELD_PREP(IRDMA_CQPSQ_AEQ_LPBLSIZE, aeq->pbl_chunk_size) |
+ FIELD_PREP(IRDMA_CQPSQ_AEQ_VMAP, aeq->virtual_map) |
+ FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
@@ -4160,29 +4159,28 @@
{
u64 temp, compl_ctx;
__le64 *aeqe;
- u16 wqe_idx;
u8 ae_src;
u8 polarity;
aeqe = IRDMA_GET_CURRENT_AEQ_ELEM(aeq);
get_64bit_val(aeqe, IRDMA_BYTE_0, &compl_ctx);
get_64bit_val(aeqe, IRDMA_BYTE_8, &temp);
- polarity = (u8)RS_64(temp, IRDMA_AEQE_VALID);
+ polarity = (u8)FIELD_GET(IRDMA_AEQE_VALID, temp);
if (aeq->polarity != polarity)
return -ENOENT;
irdma_debug_buf(aeq->dev, IRDMA_DEBUG_WQE, "AEQ_ENTRY WQE", aeqe, 16);
- ae_src = (u8)RS_64(temp, IRDMA_AEQE_AESRC);
- wqe_idx = (u16)RS_64(temp, IRDMA_AEQE_WQDESCIDX);
- info->qp_cq_id = (u32)RS_64(temp, IRDMA_AEQE_QPCQID_LOW) |
- ((u32)RS_64(temp, IRDMA_AEQE_QPCQID_HI) << 18);
- info->ae_id = (u16)RS_64(temp, IRDMA_AEQE_AECODE);
- info->tcp_state = (u8)RS_64(temp, IRDMA_AEQE_TCPSTATE);
- info->iwarp_state = (u8)RS_64(temp, IRDMA_AEQE_IWSTATE);
- info->q2_data_written = (u8)RS_64(temp, IRDMA_AEQE_Q2DATA);
- info->aeqe_overflow = (bool)RS_64(temp, IRDMA_AEQE_OVERFLOW);
+ ae_src = (u8)FIELD_GET(IRDMA_AEQE_AESRC, temp);
+ info->wqe_idx = (u16)FIELD_GET(IRDMA_AEQE_WQDESCIDX, temp);
+ info->qp_cq_id = (u32)FIELD_GET(IRDMA_AEQE_QPCQID_LOW, temp) |
+ ((u32)FIELD_GET(IRDMA_AEQE_QPCQID_HI, temp) << 18);
+ info->ae_id = (u16)FIELD_GET(IRDMA_AEQE_AECODE, temp);
+ info->tcp_state = (u8)FIELD_GET(IRDMA_AEQE_TCPSTATE, temp);
+ info->iwarp_state = (u8)FIELD_GET(IRDMA_AEQE_IWSTATE, temp);
+ info->q2_data_written = (u8)FIELD_GET(IRDMA_AEQE_Q2DATA, temp);
+ info->aeqe_overflow = (bool)FIELD_GET(IRDMA_AEQE_OVERFLOW, temp);
info->ae_src = ae_src;
switch (info->ae_id) {
@@ -4258,7 +4256,6 @@
case IRDMA_AE_SOURCE_RQ_0011:
info->qp = true;
info->rq = true;
- info->wqe_idx = wqe_idx;
info->compl_ctx = compl_ctx;
break;
case IRDMA_AE_SOURCE_CQ:
@@ -4272,7 +4269,6 @@
case IRDMA_AE_SOURCE_SQ_0111:
info->qp = true;
info->sq = true;
- info->wqe_idx = wqe_idx;
info->compl_ctx = compl_ctx;
break;
case IRDMA_AE_SOURCE_IN_WR:
@@ -4307,7 +4303,7 @@
int
irdma_sc_repost_aeq_entries(struct irdma_sc_dev *dev, u32 count)
{
- writel(count, dev->hw_regs[IRDMA_AEQALLOC]);
+ db_wr32(count, dev->aeq_alloc_db);
return 0;
}
@@ -4433,12 +4429,12 @@
hdr = ccq->cq_uk.cq_id |
FLD_LS_64(ccq->dev, (ccq->ceq_id_valid ? ccq->ceq_id : 0),
IRDMA_CQPSQ_CQ_CEQID) |
- LS_64(IRDMA_CQP_OP_DESTROY_CQ, IRDMA_CQPSQ_OPCODE) |
- LS_64(ccq->ceqe_mask, IRDMA_CQPSQ_CQ_ENCEQEMASK) |
- LS_64(ccq->ceq_id_valid, IRDMA_CQPSQ_CQ_CEQIDVALID) |
- LS_64(ccq->tph_en, IRDMA_CQPSQ_TPHEN) |
- LS_64(ccq->cq_uk.avoid_mem_cflct, IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT) |
- LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_CQ) |
+ FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, ccq->ceqe_mask) |
+ FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, ccq->ceq_id_valid) |
+ FIELD_PREP(IRDMA_CQPSQ_TPHEN, ccq->tph_en) |
+ FIELD_PREP(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT, ccq->cq_uk.avoid_mem_cflct) |
+ FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
@@ -4464,7 +4460,7 @@
* @hmc_fn_id: hmc function id
*/
int
-irdma_sc_init_iw_hmc(struct irdma_sc_dev *dev, u8 hmc_fn_id)
+irdma_sc_init_iw_hmc(struct irdma_sc_dev *dev, u16 hmc_fn_id)
{
struct irdma_hmc_info *hmc_info;
struct irdma_hmc_fpm_misc *hmc_fpm_misc;
@@ -4500,7 +4496,7 @@
* @hmc_fn_id: hmc function id
*/
static int
-irdma_sc_cfg_iw_fpm(struct irdma_sc_dev *dev, u8 hmc_fn_id)
+irdma_sc_cfg_iw_fpm(struct irdma_sc_dev *dev, u16 hmc_fn_id)
{
struct irdma_hmc_obj_info *obj_info;
__le64 *buf;
@@ -4593,27 +4589,27 @@
} else {
data = 0;
}
- data |= LS_64(info->hmc_fn_id, IRDMA_CQPSQ_UPESD_HMCFNID);
+ data |= FLD_LS_64(cqp->dev, info->hmc_fn_id, IRDMA_CQPSQ_UPESD_HMCFNID);
set_64bit_val(wqe, IRDMA_BYTE_16, data);
switch (wqe_entries) {
case 3:
set_64bit_val(wqe, IRDMA_BYTE_48,
- (LS_64(info->entry[2].cmd, IRDMA_CQPSQ_UPESD_SDCMD) |
- LS_64(1, IRDMA_CQPSQ_UPESD_ENTRY_VALID)));
+ (FIELD_PREP(IRDMA_CQPSQ_UPESD_SDCMD, info->entry[2].cmd) |
+ FIELD_PREP(IRDMA_CQPSQ_UPESD_ENTRY_VALID, 1)));
set_64bit_val(wqe, IRDMA_BYTE_56, info->entry[2].data);
/* fallthrough */
case 2:
set_64bit_val(wqe, IRDMA_BYTE_32,
- (LS_64(info->entry[1].cmd, IRDMA_CQPSQ_UPESD_SDCMD) |
- LS_64(1, IRDMA_CQPSQ_UPESD_ENTRY_VALID)));
+ (FIELD_PREP(IRDMA_CQPSQ_UPESD_SDCMD, info->entry[1].cmd) |
+ FIELD_PREP(IRDMA_CQPSQ_UPESD_ENTRY_VALID, 1)));
set_64bit_val(wqe, IRDMA_BYTE_40, info->entry[1].data);
/* fallthrough */
case 1:
set_64bit_val(wqe, IRDMA_BYTE_0,
- LS_64(info->entry[0].cmd, IRDMA_CQPSQ_UPESD_SDCMD));
+ FIELD_PREP(IRDMA_CQPSQ_UPESD_SDCMD, info->entry[0].cmd));
set_64bit_val(wqe, IRDMA_BYTE_8, info->entry[0].data);
break;
@@ -4621,9 +4617,9 @@
break;
}
- hdr = LS_64(IRDMA_CQP_OP_UPDATE_PE_SDS, IRDMA_CQPSQ_OPCODE) |
- LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID) |
- LS_64(mem_entries, IRDMA_CQPSQ_UPESD_ENTRY_COUNT);
+ hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_UPDATE_PE_SDS) |
+ FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity) |
+ FIELD_PREP(IRDMA_CQPSQ_UPESD_ENTRY_COUNT, mem_entries);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
@@ -4692,7 +4688,7 @@
*/
int
irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp *cqp, u64 scratch,
- u8 hmc_fn_id, bool post_sq,
+ u16 hmc_fn_id, bool post_sq,
bool poll_registers)
{
u64 hdr;
@@ -4704,10 +4700,11 @@
return -ENOSPC;
set_64bit_val(wqe, IRDMA_BYTE_16,
- LS_64(hmc_fn_id, IRDMA_SHMC_PAGE_ALLOCATED_HMC_FN_ID));
+ FIELD_PREP(IRDMA_SHMC_PAGE_ALLOCATED_HMC_FN_ID, hmc_fn_id));
- hdr = LS_64(IRDMA_CQP_OP_SHMC_PAGES_ALLOCATED, IRDMA_CQPSQ_OPCODE) |
- LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE,
+ IRDMA_CQP_OP_SHMC_PAGES_ALLOCATED) |
+ FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
@@ -4792,9 +4789,10 @@
temp = buf->pa;
set_64bit_val(wqe, IRDMA_BYTE_32, temp);
- temp = LS_64(cqp->polarity, IRDMA_CQPSQ_QUERY_RDMA_FEATURES_WQEVALID) |
- LS_64(buf->size, IRDMA_CQPSQ_QUERY_RDMA_FEATURES_BUF_LEN) |
- LS_64(IRDMA_CQP_OP_QUERY_RDMA_FEATURES, IRDMA_CQPSQ_UP_OP);
+ temp = FIELD_PREP(IRDMA_CQPSQ_QUERY_RDMA_FEATURES_WQEVALID,
+ cqp->polarity) |
+ FIELD_PREP(IRDMA_CQPSQ_QUERY_RDMA_FEATURES_BUF_LEN, buf->size) |
+ FIELD_PREP(IRDMA_CQPSQ_UP_OP, IRDMA_CQP_OP_QUERY_RDMA_FEATURES);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, temp);
@@ -4835,7 +4833,7 @@
goto exit;
get_64bit_val(feat_buf.va, IRDMA_BYTE_0, &temp);
- feat_cnt = (u16)RS_64(temp, IRDMA_FEATURE_CNT);
+ feat_cnt = (u16)FIELD_GET(IRDMA_FEATURE_CNT, temp);
if (feat_cnt < IRDMA_MIN_FEATURES) {
ret_code = -EINVAL;
goto exit;
@@ -4856,7 +4854,7 @@
goto exit;
get_64bit_val(feat_buf.va, IRDMA_BYTE_0, &temp);
- feat_cnt = (u16)RS_64(temp, IRDMA_FEATURE_CNT);
+ feat_cnt = (u16)FIELD_GET(IRDMA_FEATURE_CNT, temp);
if (feat_cnt < IRDMA_MIN_FEATURES) {
ret_code = -EINVAL;
goto exit;
@@ -4869,7 +4867,7 @@
for (byte_idx = 0, feat_idx = 0; feat_idx < min(feat_cnt, (u16)IRDMA_MAX_FEATURES);
feat_idx++, byte_idx += 8) {
get_64bit_val(feat_buf.va, byte_idx, &temp);
- feat_type = RS_64(temp, IRDMA_FEATURE_TYPE);
+ feat_type = FIELD_GET(IRDMA_FEATURE_TYPE, temp);
dev->feature_info[feat_type] = temp;
}
exit:
@@ -4990,8 +4988,8 @@
hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].max_cnt;
hmc_info->hmc_obj[IRDMA_HMC_IW_ARP].cnt =
hmc_info->hmc_obj[IRDMA_HMC_IW_ARP].max_cnt;
-
- hmc_info->hmc_obj[IRDMA_HMC_IW_APBVT_ENTRY].cnt = 1;
+ if (dev->hw_attrs.uk_attrs.hw_rev <= IRDMA_GEN_2)
+ hmc_info->hmc_obj[IRDMA_HMC_IW_APBVT_ENTRY].cnt = 1;
while (irdma_q1_cnt(dev, hmc_info, qpwanted) > hmc_info->hmc_obj[IRDMA_HMC_IW_Q1].max_cnt)
qpwanted /= 2;
@@ -5109,7 +5107,7 @@
mem_size = sizeof(struct irdma_hmc_sd_entry) *
(hmc_info->sd_table.sd_cnt + hmc_info->first_sd_index + 1);
virt_mem.size = mem_size;
- virt_mem.va = kzalloc(virt_mem.size, GFP_ATOMIC);
+ virt_mem.va = kzalloc(virt_mem.size, GFP_KERNEL);
if (!virt_mem.va) {
irdma_debug(dev, IRDMA_DEBUG_HMC,
"failed to allocate memory for sd_entry buffer\n");
@@ -5144,7 +5142,6 @@
status = irdma_sc_aeq_destroy(pcmdinfo->in.u.aeq_destroy.aeq,
pcmdinfo->in.u.aeq_destroy.scratch,
pcmdinfo->post_sq);
-
break;
case IRDMA_OP_CEQ_CREATE:
status = irdma_sc_ceq_create(pcmdinfo->in.u.ceq_create.ceq,
@@ -5410,9 +5407,6 @@
int status = 0;
unsigned long flags;
- if (dev->no_cqp)
- return -EFAULT;
-
spin_lock_irqsave(&dev->cqp_lock, flags);
if (list_empty(&dev->cqp_cmd_head) && !irdma_cqp_ring_full(dev->cqp))
status = irdma_exec_cqp_cmd(dev, pcmdinfo);
@@ -5455,9 +5449,10 @@
irdma_cfg_aeq(struct irdma_sc_dev *dev, u32 idx, bool enable)
{
u32 reg_val;
- reg_val = enable ? IRDMA_PFINT_AEQCTL_CAUSE_ENA_M : 0;
- reg_val |= (idx << IRDMA_PFINT_AEQCTL_MSIX_INDX_S) |
- IRDMA_PFINT_AEQCTL_ITR_INDX_M;
+ reg_val = FIELD_PREP(IRDMA_PFINT_AEQCTL_CAUSE_ENA, enable) |
+ FIELD_PREP(IRDMA_PFINT_AEQCTL_MSIX_INDX, idx) |
+ FIELD_PREP(IRDMA_PFINT_AEQCTL_ITR_INDX, IRDMA_IDX_NOITR);
+
writel(reg_val, dev->hw_regs[IRDMA_PFINT_AEQCTL]);
}
@@ -5514,13 +5509,11 @@
/**
* irdma_sc_dev_init - Initialize control part of device
- * @ver: version
* @dev: Device pointer
* @info: Device init info
*/
int
-irdma_sc_dev_init(enum irdma_vers ver, struct irdma_sc_dev *dev,
- struct irdma_device_init_info *info)
+irdma_sc_dev_init(struct irdma_sc_dev *dev, struct irdma_device_init_info *info)
{
u32 val;
int ret_code = 0;
@@ -5562,14 +5555,13 @@
dev->hw_attrs.max_sleep_count = IRDMA_SLEEP_COUNT;
dev->hw_attrs.max_cqp_compl_wait_time_ms = CQP_COMPL_WAIT_TIME_MS;
- dev->hw_attrs.uk_attrs.hw_rev = ver;
irdma_sc_init_hw(dev);
if (irdma_wait_pe_ready(dev))
return -ETIMEDOUT;
val = readl(dev->hw_regs[IRDMA_GLPCI_LBARCTRL]);
- db_size = (u8)RS_32(val, IRDMA_GLPCI_LBARCTRL_PE_DB_SIZE);
+ db_size = (u8)FIELD_GET(IRDMA_GLPCI_LBARCTRL_PE_DB_SIZE, val);
if (db_size != IRDMA_PE_DB_SIZE_4M && db_size != IRDMA_PE_DB_SIZE_8M) {
irdma_debug(dev, IRDMA_DEBUG_DEV,
"RDMA PE doorbell is not enabled in CSR val 0x%x db_size=%d\n",
diff --git a/sys/dev/irdma/irdma_defs.h b/sys/dev/irdma/irdma_defs.h
--- a/sys/dev/irdma/irdma_defs.h
+++ b/sys/dev/irdma/irdma_defs.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2021 Intel Corporation
+ * Copyright (c) 2015 - 2022 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -48,83 +48,12 @@
#define IRDMA_PE_DB_SIZE_4M 1
#define IRDMA_PE_DB_SIZE_8M 2
-#define IRDMA_DDP_VER 1
-#define IRDMA_RDMAP_VER 1
-
-#define IRDMA_RDMA_MODE_RDMAC 0
-#define IRDMA_RDMA_MODE_IETF 1
-
-#define IRDMA_STAG_STATE_INVALID 0
-#define IRDMA_STAG_STATE_VALID 1
-#define IRDMA_STAG_TYPE_SHARED 0
-#define IRDMA_STAG_TYPE_NONSHARED 1
-
-#define QS_HANDLE_UNKNOWN 0xffff
-#define USER_PRI_UNKNOWN 0xff
-
-#define IRDMA_INVALID_WQE_INDEX 0xffffffff
-
-#define IRDMA_CQP_SW_SQSIZE_8 8
-#define IRDMA_CQP_SW_SQSIZE_16 16
-#define IRDMA_CQP_SW_SQSIZE_32 32
-#define IRDMA_CQP_SW_SQSIZE_64 64
-#define IRDMA_CQP_SW_SQSIZE_128 128
-#define IRDMA_CQP_SW_SQSIZE_256 256
-#define IRDMA_CQP_SW_SQSIZE_512 512
-#define IRDMA_CQP_SW_SQSIZE_1024 1024
-
-#define IRDMA_CQP_HW_SQSIZE_4 1
-#define IRDMA_CQP_HW_SQSIZE_8 2
-#define IRDMA_CQP_HW_SQSIZE_16 3
-#define IRDMA_CQP_HW_SQSIZE_32 4
-#define IRDMA_CQP_HW_SQSIZE_64 5
-#define IRDMA_CQP_HW_SQSIZE_128 6
-#define IRDMA_CQP_HW_SQSIZE_256 7
-#define IRDMA_CQP_HW_SQSIZE_512 8
-#define IRDMA_CQP_HW_SQSIZE_1024 9
-#define IRDMA_CQP_HW_SQSIZE_2048 10
-
-/* WQE size considering 32 bytes per WQE*/
-#define IRDMAQP_SW_WQSIZE_8 8 /* 256 bytes */
-#define IRDMAQP_SW_WQSIZE_16 16 /* 512 bytes */
-#define IRDMAQP_SW_WQSIZE_32 32 /* 1024 bytes */
-#define IRDMAQP_SW_WQSIZE_64 64 /* 2048 bytes */
-#define IRDMAQP_SW_WQSIZE_128 128 /* 4096 bytes */
-#define IRDMAQP_SW_WQSIZE_256 256 /* 8192 bytes */
-#define IRDMAQP_SW_WQSIZE_512 512 /* 16384 bytes */
-#define IRDMAQP_SW_WQSIZE_1024 1024 /* 32768 bytes */
-#define IRDMAQP_SW_WQSIZE_2048 2048 /* 65536 bytes */
-#define IRDMAQP_SW_WQSIZE_4096 4096 /* 131072 bytes */
-#define IRDMAQP_SW_WQSIZE_8192 8192 /* 262144 bytes */
-#define IRDMAQP_SW_WQSIZE_16384 16384 /* 524288 bytes */
-#define IRDMAQP_SW_WQSIZE_32768 32768 /* 1048576 bytes */
-
-#define IRDMAQP_HW_WQSIZE_8 1
-#define IRDMAQP_HW_WQSIZE_16 2
-#define IRDMAQP_HW_WQSIZE_32 3
-#define IRDMAQP_HW_WQSIZE_64 4
-#define IRDMAQP_HW_WQSIZE_128 5
-#define IRDMAQP_HW_WQSIZE_256 6
-#define IRDMAQP_HW_WQSIZE_512 7
-#define IRDMAQP_HW_WQSIZE_1024 8
-#define IRDMAQP_HW_WQSIZE_2048 9
-#define IRDMAQP_HW_WQSIZE_4096 10
-#define IRDMAQP_HW_WQSIZE_8192 11
-#define IRDMAQP_HW_WQSIZE_16384 12
-#define IRDMAQP_HW_WQSIZE_32768 13
-
#define IRDMA_IRD_HW_SIZE_4 0
#define IRDMA_IRD_HW_SIZE_16 1
#define IRDMA_IRD_HW_SIZE_64 2
#define IRDMA_IRD_HW_SIZE_128 3
#define IRDMA_IRD_HW_SIZE_256 4
-enum irdma_protocol_used {
- IRDMA_ANY_PROTOCOL = 0,
- IRDMA_IWARP_PROTOCOL_ONLY = 1,
- IRDMA_ROCE_PROTOCOL_ONLY = 2,
-};
-
#define IRDMA_QP_STATE_INVALID 0
#define IRDMA_QP_STATE_IDLE 1
#define IRDMA_QP_STATE_RTS 2
@@ -245,6 +174,7 @@
#define IRDMA_QP_SW_MAX_WQ_QUANTA 32768
#define IRDMA_QP_SW_MAX_SQ_QUANTA 32768
#define IRDMA_QP_SW_MAX_RQ_QUANTA 32768
+
#define IRDMA_MAX_QP_WRS(max_quanta_per_wr) \
((IRDMA_QP_SW_MAX_WQ_QUANTA - IRDMA_SQ_RSVD) / (max_quanta_per_wr))
@@ -263,7 +193,6 @@
#define IRDMA_CQE_QTYPE_RQ 0
#define IRDMA_CQE_QTYPE_SQ 1
-#define IRDMA_QP_SW_MIN_WQSIZE 8u /* in WRs*/
#define IRDMA_QP_WQE_MIN_SIZE 32
#define IRDMA_QP_WQE_MAX_SIZE 256
#define IRDMA_QP_WQE_MIN_QUANTA 1
@@ -273,9 +202,11 @@
#define IRDMA_SQ_RSVD 258
#define IRDMA_RQ_RSVD 1
-#define IRDMA_FEATURE_RTS_AE 1ULL
-#define IRDMA_FEATURE_CQ_RESIZE 2ULL
-#define IRDMA_FEATURE_RELAX_RQ_ORDER 4ULL
+#define IRDMA_FEATURE_RTS_AE BIT_ULL(0)
+#define IRDMA_FEATURE_CQ_RESIZE BIT_ULL(1)
+#define IRDMA_FEATURE_RELAX_RQ_ORDER BIT_ULL(2)
+#define IRDMA_FEATURE_64_BYTE_CQE BIT_ULL(5)
+
#define IRDMAQP_OP_RDMA_WRITE 0x00
#define IRDMAQP_OP_RDMA_READ 0x01
#define IRDMAQP_OP_RDMA_SEND 0x03
@@ -342,7 +273,7 @@
IRDMA_OP_DELETE_LOCAL_MAC_ENTRY = 49,
IRDMA_OP_CQ_MODIFY = 50,
- /* Must be last entry*/
+ /* Must be last entry */
IRDMA_MAX_CQP_OPS = 51,
};
@@ -394,105 +325,22 @@
#define IRDMA_CQP_OP_GATHER_STATS 0x2e
#define IRDMA_CQP_OP_UP_MAP 0x2f
-/* Async Events codes */
-#define IRDMA_AE_AMP_UNALLOCATED_STAG 0x0102
-#define IRDMA_AE_AMP_INVALID_STAG 0x0103
-#define IRDMA_AE_AMP_BAD_QP 0x0104
-#define IRDMA_AE_AMP_BAD_PD 0x0105
-#define IRDMA_AE_AMP_BAD_STAG_KEY 0x0106
-#define IRDMA_AE_AMP_BAD_STAG_INDEX 0x0107
-#define IRDMA_AE_AMP_BOUNDS_VIOLATION 0x0108
-#define IRDMA_AE_AMP_RIGHTS_VIOLATION 0x0109
-#define IRDMA_AE_AMP_TO_WRAP 0x010a
-#define IRDMA_AE_AMP_FASTREG_VALID_STAG 0x010c
-#define IRDMA_AE_AMP_FASTREG_MW_STAG 0x010d
-#define IRDMA_AE_AMP_FASTREG_INVALID_RIGHTS 0x010e
-#define IRDMA_AE_AMP_FASTREG_INVALID_LENGTH 0x0110
-#define IRDMA_AE_AMP_INVALIDATE_SHARED 0x0111
-#define IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS 0x0112
-#define IRDMA_AE_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS 0x0113
-#define IRDMA_AE_AMP_MWBIND_VALID_STAG 0x0114
-#define IRDMA_AE_AMP_MWBIND_OF_MR_STAG 0x0115
-#define IRDMA_AE_AMP_MWBIND_TO_ZERO_BASED_STAG 0x0116
-#define IRDMA_AE_AMP_MWBIND_TO_MW_STAG 0x0117
-#define IRDMA_AE_AMP_MWBIND_INVALID_RIGHTS 0x0118
-#define IRDMA_AE_AMP_MWBIND_INVALID_BOUNDS 0x0119
-#define IRDMA_AE_AMP_MWBIND_TO_INVALID_PARENT 0x011a
-#define IRDMA_AE_AMP_MWBIND_BIND_DISABLED 0x011b
-#define IRDMA_AE_PRIV_OPERATION_DENIED 0x011c
-#define IRDMA_AE_AMP_INVALIDATE_TYPE1_MW 0x011d
-#define IRDMA_AE_AMP_MWBIND_ZERO_BASED_TYPE1_MW 0x011e
-#define IRDMA_AE_AMP_FASTREG_INVALID_PBL_HPS_CFG 0x011f
-#define IRDMA_AE_AMP_MWBIND_WRONG_TYPE 0x0120
-#define IRDMA_AE_AMP_FASTREG_PBLE_MISMATCH 0x0121
-#define IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG 0x0132
-#define IRDMA_AE_UDA_XMIT_BAD_PD 0x0133
-#define IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT 0x0134
-#define IRDMA_AE_UDA_L4LEN_INVALID 0x0135
-#define IRDMA_AE_BAD_CLOSE 0x0201
-#define IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE 0x0202
-#define IRDMA_AE_CQ_OPERATION_ERROR 0x0203
-#define IRDMA_AE_RDMA_READ_WHILE_ORD_ZERO 0x0205
-#define IRDMA_AE_STAG_ZERO_INVALID 0x0206
-#define IRDMA_AE_IB_RREQ_AND_Q1_FULL 0x0207
-#define IRDMA_AE_IB_INVALID_REQUEST 0x0208
-#define IRDMA_AE_WQE_UNEXPECTED_OPCODE 0x020a
-#define IRDMA_AE_WQE_INVALID_PARAMETER 0x020b
-#define IRDMA_AE_WQE_INVALID_FRAG_DATA 0x020c
-#define IRDMA_AE_IB_REMOTE_ACCESS_ERROR 0x020d
-#define IRDMA_AE_IB_REMOTE_OP_ERROR 0x020e
-#define IRDMA_AE_WQE_LSMM_TOO_LONG 0x0220
-#define IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN 0x0301
-#define IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER 0x0303
-#define IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION 0x0304
-#define IRDMA_AE_DDP_UBE_INVALID_MO 0x0305
-#define IRDMA_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE 0x0306
-#define IRDMA_AE_DDP_UBE_INVALID_QN 0x0307
-#define IRDMA_AE_DDP_NO_L_BIT 0x0308
-#define IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION 0x0311
-#define IRDMA_AE_RDMAP_ROE_UNEXPECTED_OPCODE 0x0312
-#define IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST 0x0313
-#define IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP 0x0314
-#define IRDMA_AE_ROCE_RSP_LENGTH_ERROR 0x0316
-#define IRDMA_AE_ROCE_EMPTY_MCG 0x0380
-#define IRDMA_AE_ROCE_BAD_MC_IP_ADDR 0x0381
-#define IRDMA_AE_ROCE_BAD_MC_QPID 0x0382
-#define IRDMA_AE_MCG_QP_PROTOCOL_MISMATCH 0x0383
-#define IRDMA_AE_INVALID_ARP_ENTRY 0x0401
-#define IRDMA_AE_INVALID_TCP_OPTION_RCVD 0x0402
-#define IRDMA_AE_STALE_ARP_ENTRY 0x0403
-#define IRDMA_AE_INVALID_AH_ENTRY 0x0406
-#define IRDMA_AE_LLP_CLOSE_COMPLETE 0x0501
-#define IRDMA_AE_LLP_CONNECTION_RESET 0x0502
-#define IRDMA_AE_LLP_FIN_RECEIVED 0x0503
-#define IRDMA_AE_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH 0x0504
-#define IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR 0x0505
-#define IRDMA_AE_LLP_SEGMENT_TOO_SMALL 0x0507
-#define IRDMA_AE_LLP_SYN_RECEIVED 0x0508
-#define IRDMA_AE_LLP_TERMINATE_RECEIVED 0x0509
-#define IRDMA_AE_LLP_TOO_MANY_RETRIES 0x050a
-#define IRDMA_AE_LLP_TOO_MANY_KEEPALIVE_RETRIES 0x050b
-#define IRDMA_AE_LLP_DOUBT_REACHABILITY 0x050c
-#define IRDMA_AE_LLP_CONNECTION_ESTABLISHED 0x050e
-#define IRDMA_AE_RESOURCE_EXHAUSTION 0x0520
-#define IRDMA_AE_RESET_SENT 0x0601
-#define IRDMA_AE_TERMINATE_SENT 0x0602
-#define IRDMA_AE_RESET_NOT_SENT 0x0603
-#define IRDMA_AE_LCE_QP_CATASTROPHIC 0x0700
-#define IRDMA_AE_LCE_FUNCTION_CATASTROPHIC 0x0701
-#define IRDMA_AE_LCE_CQ_CATASTROPHIC 0x0702
-#define IRDMA_AE_QP_SUSPEND_COMPLETE 0x0900
-
#ifndef LS_64_1
#define LS_64_1(val, bits) ((u64)(uintptr_t)(val) << (bits))
#define RS_64_1(val, bits) ((u64)(uintptr_t)(val) >> (bits))
#define LS_32_1(val, bits) ((u32)((val) << (bits)))
#define RS_32_1(val, bits) ((u32)((val) >> (bits)))
#endif
-#define LS_64(val, field) (((u64)(val) << field ## _S) & (field ## _M))
-#define RS_64(val, field) ((u64)((val) & field ## _M) >> field ## _S)
-#define LS_32(val, field) (((val) << field ## _S) & (field ## _M))
-#define RS_32(val, field) (((val) & field ## _M) >> field ## _S)
+#ifndef GENMASK_ULL
+#define GENMASK_ULL(high, low) ((0xFFFFFFFFFFFFFFFFULL >> (64ULL - ((high) - (low) + 1ULL))) << (low))
+#endif /* GENMASK_ULL */
+#ifndef GENMASK
+#define GENMASK(high, low) ((0xFFFFFFFFUL >> (32UL - ((high) - (low) + 1UL))) << (low))
+#endif /* GENMASK */
+#ifndef FIELD_PREP
+#define FIELD_PREP(mask, val) (((u64)(val) << mask##_S) & (mask))
+#define FIELD_GET(mask, val) (((val) & mask) >> mask##_S)
+#endif /* FIELD_PREP */
#define FLD_LS_64(dev, val, field) \
(((u64)(val) << (dev)->hw_shifts[field ## _S]) & (dev)->hw_masks[field ## _M])
@@ -511,954 +359,594 @@
#define IRDMA_MAX_STATS_64 0xffffffffffffffffULL
#define IRDMA_MAX_CQ_READ_THRESH 0x3FFFF
-/* ILQ CQP hash table fields */
#define IRDMA_CQPSQ_QHASH_VLANID_S 32
-#define IRDMA_CQPSQ_QHASH_VLANID_M \
- ((u64)0xfff << IRDMA_CQPSQ_QHASH_VLANID_S)
-
+#define IRDMA_CQPSQ_QHASH_VLANID GENMASK_ULL(43, 32)
#define IRDMA_CQPSQ_QHASH_QPN_S 32
-#define IRDMA_CQPSQ_QHASH_QPN_M \
- ((u64)0x3ffff << IRDMA_CQPSQ_QHASH_QPN_S)
-
+#define IRDMA_CQPSQ_QHASH_QPN GENMASK_ULL(49, 32)
#define IRDMA_CQPSQ_QHASH_QS_HANDLE_S 0
-#define IRDMA_CQPSQ_QHASH_QS_HANDLE_M ((u64)0x3ff << IRDMA_CQPSQ_QHASH_QS_HANDLE_S)
-
+#define IRDMA_CQPSQ_QHASH_QS_HANDLE GENMASK_ULL(9, 0)
#define IRDMA_CQPSQ_QHASH_SRC_PORT_S 16
-#define IRDMA_CQPSQ_QHASH_SRC_PORT_M \
- ((u64)0xffff << IRDMA_CQPSQ_QHASH_SRC_PORT_S)
-
+#define IRDMA_CQPSQ_QHASH_SRC_PORT GENMASK_ULL(31, 16)
#define IRDMA_CQPSQ_QHASH_DEST_PORT_S 0
-#define IRDMA_CQPSQ_QHASH_DEST_PORT_M \
- ((u64)0xffff << IRDMA_CQPSQ_QHASH_DEST_PORT_S)
-
+#define IRDMA_CQPSQ_QHASH_DEST_PORT GENMASK_ULL(15, 0)
#define IRDMA_CQPSQ_QHASH_ADDR0_S 32
-#define IRDMA_CQPSQ_QHASH_ADDR0_M \
- ((u64)0xffffffff << IRDMA_CQPSQ_QHASH_ADDR0_S)
-
+#define IRDMA_CQPSQ_QHASH_ADDR0 GENMASK_ULL(63, 32)
#define IRDMA_CQPSQ_QHASH_ADDR1_S 0
-#define IRDMA_CQPSQ_QHASH_ADDR1_M \
- ((u64)0xffffffff << IRDMA_CQPSQ_QHASH_ADDR1_S)
-
+#define IRDMA_CQPSQ_QHASH_ADDR1 GENMASK_ULL(31, 0)
#define IRDMA_CQPSQ_QHASH_ADDR2_S 32
-#define IRDMA_CQPSQ_QHASH_ADDR2_M \
- ((u64)0xffffffff << IRDMA_CQPSQ_QHASH_ADDR2_S)
-
+#define IRDMA_CQPSQ_QHASH_ADDR2 GENMASK_ULL(63, 32)
#define IRDMA_CQPSQ_QHASH_ADDR3_S 0
-#define IRDMA_CQPSQ_QHASH_ADDR3_M \
- ((u64)0xffffffff << IRDMA_CQPSQ_QHASH_ADDR3_S)
-
+#define IRDMA_CQPSQ_QHASH_ADDR3 GENMASK_ULL(31, 0)
#define IRDMA_CQPSQ_QHASH_WQEVALID_S 63
-#define IRDMA_CQPSQ_QHASH_WQEVALID_M \
- BIT_ULL(IRDMA_CQPSQ_QHASH_WQEVALID_S)
+#define IRDMA_CQPSQ_QHASH_WQEVALID BIT_ULL(63)
#define IRDMA_CQPSQ_QHASH_OPCODE_S 32
-#define IRDMA_CQPSQ_QHASH_OPCODE_M \
- ((u64)0x3f << IRDMA_CQPSQ_QHASH_OPCODE_S)
-
+#define IRDMA_CQPSQ_QHASH_OPCODE GENMASK_ULL(37, 32)
#define IRDMA_CQPSQ_QHASH_MANAGE_S 61
-#define IRDMA_CQPSQ_QHASH_MANAGE_M \
- ((u64)0x3 << IRDMA_CQPSQ_QHASH_MANAGE_S)
-
+#define IRDMA_CQPSQ_QHASH_MANAGE GENMASK_ULL(62, 61)
#define IRDMA_CQPSQ_QHASH_IPV4VALID_S 60
-#define IRDMA_CQPSQ_QHASH_IPV4VALID_M \
- BIT_ULL(IRDMA_CQPSQ_QHASH_IPV4VALID_S)
-
+#define IRDMA_CQPSQ_QHASH_IPV4VALID BIT_ULL(60)
#define IRDMA_CQPSQ_QHASH_VLANVALID_S 59
-#define IRDMA_CQPSQ_QHASH_VLANVALID_M \
- BIT_ULL(IRDMA_CQPSQ_QHASH_VLANVALID_S)
-
+#define IRDMA_CQPSQ_QHASH_VLANVALID BIT_ULL(59)
#define IRDMA_CQPSQ_QHASH_ENTRYTYPE_S 42
-#define IRDMA_CQPSQ_QHASH_ENTRYTYPE_M \
- ((u64)0x7 << IRDMA_CQPSQ_QHASH_ENTRYTYPE_S)
-
-/* Stats */
+#define IRDMA_CQPSQ_QHASH_ENTRYTYPE GENMASK_ULL(44, 42)
#define IRDMA_CQPSQ_STATS_WQEVALID_S 63
-#define IRDMA_CQPSQ_STATS_WQEVALID_M \
- BIT_ULL(IRDMA_CQPSQ_STATS_WQEVALID_S)
-
+#define IRDMA_CQPSQ_STATS_WQEVALID BIT_ULL(63)
#define IRDMA_CQPSQ_STATS_ALLOC_INST_S 62
-#define IRDMA_CQPSQ_STATS_ALLOC_INST_M \
- BIT_ULL(IRDMA_CQPSQ_STATS_ALLOC_INST_S)
-
+#define IRDMA_CQPSQ_STATS_ALLOC_INST BIT_ULL(62)
#define IRDMA_CQPSQ_STATS_USE_HMC_FCN_INDEX_S 60
-#define IRDMA_CQPSQ_STATS_USE_HMC_FCN_INDEX_M \
- BIT_ULL(IRDMA_CQPSQ_STATS_USE_HMC_FCN_INDEX_S)
-
+#define IRDMA_CQPSQ_STATS_USE_HMC_FCN_INDEX BIT_ULL(60)
#define IRDMA_CQPSQ_STATS_USE_INST_S 61
-#define IRDMA_CQPSQ_STATS_USE_INST_M \
- BIT_ULL(IRDMA_CQPSQ_STATS_USE_INST_S)
-
+#define IRDMA_CQPSQ_STATS_USE_INST BIT_ULL(61)
#define IRDMA_CQPSQ_STATS_OP_S 32
-#define IRDMA_CQPSQ_STATS_OP_M \
- ((u64)0x3f << IRDMA_CQPSQ_STATS_OP_S)
-
+#define IRDMA_CQPSQ_STATS_OP GENMASK_ULL(37, 32)
#define IRDMA_CQPSQ_STATS_INST_INDEX_S 0
-#define IRDMA_CQPSQ_STATS_INST_INDEX_M \
- ((u64)0x7f << IRDMA_CQPSQ_STATS_INST_INDEX_S)
-
+#define IRDMA_CQPSQ_STATS_INST_INDEX GENMASK_ULL(6, 0)
#define IRDMA_CQPSQ_STATS_HMC_FCN_INDEX_S 0
-#define IRDMA_CQPSQ_STATS_HMC_FCN_INDEX_M \
- ((u64)0x3f << IRDMA_CQPSQ_STATS_HMC_FCN_INDEX_S)
-
-/* WS - Work Scheduler */
+#define IRDMA_CQPSQ_STATS_HMC_FCN_INDEX GENMASK_ULL(15, 0)
#define IRDMA_CQPSQ_WS_WQEVALID_S 63
-#define IRDMA_CQPSQ_WS_WQEVALID_M \
- BIT_ULL(IRDMA_CQPSQ_WS_WQEVALID_S)
-
+#define IRDMA_CQPSQ_WS_WQEVALID BIT_ULL(63)
#define IRDMA_CQPSQ_WS_NODEOP_S 52
-#define IRDMA_CQPSQ_WS_NODEOP_M \
- ((u64)0x3 << IRDMA_CQPSQ_WS_NODEOP_S)
+#define IRDMA_CQPSQ_WS_NODEOP GENMASK_ULL(55, 52)
#define IRDMA_CQPSQ_WS_ENABLENODE_S 62
-#define IRDMA_CQPSQ_WS_ENABLENODE_M \
- BIT_ULL(IRDMA_CQPSQ_WS_ENABLENODE_S)
-
+#define IRDMA_CQPSQ_WS_ENABLENODE BIT_ULL(62)
#define IRDMA_CQPSQ_WS_NODETYPE_S 61
-#define IRDMA_CQPSQ_WS_NODETYPE_M \
- BIT_ULL(IRDMA_CQPSQ_WS_NODETYPE_S)
-
+#define IRDMA_CQPSQ_WS_NODETYPE BIT_ULL(61)
#define IRDMA_CQPSQ_WS_PRIOTYPE_S 59
-#define IRDMA_CQPSQ_WS_PRIOTYPE_M \
- ((u64)0x3 << IRDMA_CQPSQ_WS_PRIOTYPE_S)
-
+#define IRDMA_CQPSQ_WS_PRIOTYPE GENMASK_ULL(60, 59)
#define IRDMA_CQPSQ_WS_TC_S 56
-#define IRDMA_CQPSQ_WS_TC_M \
- ((u64)0x7 << IRDMA_CQPSQ_WS_TC_S)
-
+#define IRDMA_CQPSQ_WS_TC GENMASK_ULL(58, 56)
#define IRDMA_CQPSQ_WS_VMVFTYPE_S 54
-#define IRDMA_CQPSQ_WS_VMVFTYPE_M \
- ((u64)0x3 << IRDMA_CQPSQ_WS_VMVFTYPE_S)
-
+#define IRDMA_CQPSQ_WS_VMVFTYPE GENMASK_ULL(55, 54)
#define IRDMA_CQPSQ_WS_VMVFNUM_S 42
-#define IRDMA_CQPSQ_WS_VMVFNUM_M \
- ((u64)0x3ff << IRDMA_CQPSQ_WS_VMVFNUM_S)
-
+#define IRDMA_CQPSQ_WS_VMVFNUM GENMASK_ULL(51, 42)
#define IRDMA_CQPSQ_WS_OP_S 32
-#define IRDMA_CQPSQ_WS_OP_M \
- ((u64)0x3f << IRDMA_CQPSQ_WS_OP_S)
-
+#define IRDMA_CQPSQ_WS_OP GENMASK_ULL(37, 32)
#define IRDMA_CQPSQ_WS_PARENTID_S 16
-#define IRDMA_CQPSQ_WS_PARENTID_M \
- ((u64)0x3ff << IRDMA_CQPSQ_WS_PARENTID_S)
-
+#define IRDMA_CQPSQ_WS_PARENTID GENMASK_ULL(25, 16)
#define IRDMA_CQPSQ_WS_NODEID_S 0
-#define IRDMA_CQPSQ_WS_NODEID_M \
- ((u64)0x3ff << IRDMA_CQPSQ_WS_NODEID_S)
-
+#define IRDMA_CQPSQ_WS_NODEID GENMASK_ULL(9, 0)
#define IRDMA_CQPSQ_WS_VSI_S 48
-#define IRDMA_CQPSQ_WS_VSI_M \
- ((u64)0x3ff << IRDMA_CQPSQ_WS_VSI_S)
-
+#define IRDMA_CQPSQ_WS_VSI GENMASK_ULL(57, 48)
#define IRDMA_CQPSQ_WS_WEIGHT_S 32
-#define IRDMA_CQPSQ_WS_WEIGHT_M \
- ((u64)0x7f << IRDMA_CQPSQ_WS_WEIGHT_S)
+#define IRDMA_CQPSQ_WS_WEIGHT GENMASK_ULL(38, 32)
-/* UP to UP mapping */
#define IRDMA_CQPSQ_UP_WQEVALID_S 63
-#define IRDMA_CQPSQ_UP_WQEVALID_M \
- BIT_ULL(IRDMA_CQPSQ_UP_WQEVALID_S)
-
+#define IRDMA_CQPSQ_UP_WQEVALID BIT_ULL(63)
#define IRDMA_CQPSQ_UP_USEVLAN_S 62
-#define IRDMA_CQPSQ_UP_USEVLAN_M \
- BIT_ULL(IRDMA_CQPSQ_UP_USEVLAN_S)
-
+#define IRDMA_CQPSQ_UP_USEVLAN BIT_ULL(62)
#define IRDMA_CQPSQ_UP_USEOVERRIDE_S 61
-#define IRDMA_CQPSQ_UP_USEOVERRIDE_M \
- BIT_ULL(IRDMA_CQPSQ_UP_USEOVERRIDE_S)
-
+#define IRDMA_CQPSQ_UP_USEOVERRIDE BIT_ULL(61)
#define IRDMA_CQPSQ_UP_OP_S 32
-#define IRDMA_CQPSQ_UP_OP_M \
- ((u64)0x3f << IRDMA_CQPSQ_UP_OP_S)
-
+#define IRDMA_CQPSQ_UP_OP GENMASK_ULL(37, 32)
#define IRDMA_CQPSQ_UP_HMCFCNIDX_S 0
-#define IRDMA_CQPSQ_UP_HMCFCNIDX_M \
- ((u64)0x3f << IRDMA_CQPSQ_UP_HMCFCNIDX_S)
-
+#define IRDMA_CQPSQ_UP_HMCFCNIDX GENMASK_ULL(5, 0)
#define IRDMA_CQPSQ_UP_CNPOVERRIDE_S 32
-#define IRDMA_CQPSQ_UP_CNPOVERRIDE_M \
- ((u64)0x3f << IRDMA_CQPSQ_UP_CNPOVERRIDE_S)
-
-/* Query RDMA features*/
+#define IRDMA_CQPSQ_UP_CNPOVERRIDE GENMASK_ULL(37, 32)
#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_WQEVALID_S 63
-#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_WQEVALID_M \
- BIT_ULL(IRDMA_CQPSQ_QUERY_RDMA_FEATURES_WQEVALID_S)
-
+#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_WQEVALID BIT_ULL(63)
#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_BUF_LEN_S 0
-#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_BUF_LEN_M \
- ((u64)0xffffffff << IRDMA_CQPSQ_QUERY_RDMA_FEATURES_BUF_LEN_S)
-
+#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_BUF_LEN GENMASK_ULL(31, 0)
#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_OP_S 32
-#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_OP_M \
- ((u64)0x3f << IRDMA_CQPSQ_QUERY_RDMA_FEATURES_OP_S)
-
+#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_OP GENMASK_ULL(37, 32)
#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_HW_MODEL_USED_S 32
-#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_HW_MODEL_USED_M \
- (0xffffULL << IRDMA_CQPSQ_QUERY_RDMA_FEATURES_HW_MODEL_USED_S)
-
+#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_HW_MODEL_USED GENMASK_ULL(47, 32)
#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_HW_MAJOR_VERSION_S 16
-#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_HW_MAJOR_VERSION_M \
- (0xffULL << IRDMA_CQPSQ_QUERY_RDMA_FEATURES_HW_MAJOR_VERSION_S)
-
+#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_HW_MAJOR_VERSION GENMASK_ULL(23, 16)
#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_HW_MINOR_VERSION_S 0
-#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_HW_MINOR_VERSION_M \
- (0xffULL << IRDMA_CQPSQ_QUERY_RDMA_FEATURES_HW_MINOR_VERSION_S)
-
-/* CQP Host Context */
+#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_HW_MINOR_VERSION GENMASK_ULL(7, 0)
#define IRDMA_CQPHC_SQSIZE_S 8
-#define IRDMA_CQPHC_SQSIZE_M (0xfULL << IRDMA_CQPHC_SQSIZE_S)
-
+#define IRDMA_CQPHC_SQSIZE GENMASK_ULL(11, 8)
#define IRDMA_CQPHC_DISABLE_PFPDUS_S 1
-#define IRDMA_CQPHC_DISABLE_PFPDUS_M BIT_ULL(IRDMA_CQPHC_DISABLE_PFPDUS_S)
-
+#define IRDMA_CQPHC_DISABLE_PFPDUS BIT_ULL(1)
#define IRDMA_CQPHC_ROCEV2_RTO_POLICY_S 2
-#define IRDMA_CQPHC_ROCEV2_RTO_POLICY_M BIT_ULL(IRDMA_CQPHC_ROCEV2_RTO_POLICY_S)
-
+#define IRDMA_CQPHC_ROCEV2_RTO_POLICY BIT_ULL(2)
#define IRDMA_CQPHC_PROTOCOL_USED_S 3
-#define IRDMA_CQPHC_PROTOCOL_USED_M (0x3ULL << IRDMA_CQPHC_PROTOCOL_USED_S)
-
+#define IRDMA_CQPHC_PROTOCOL_USED GENMASK_ULL(4, 3)
#define IRDMA_CQPHC_MIN_RATE_S 48
-#define IRDMA_CQPHC_MIN_RATE_M (0xfULL << IRDMA_CQPHC_MIN_RATE_S)
-
+#define IRDMA_CQPHC_MIN_RATE GENMASK_ULL(51, 48)
#define IRDMA_CQPHC_MIN_DEC_FACTOR_S 56
-#define IRDMA_CQPHC_MIN_DEC_FACTOR_M (0xfULL << IRDMA_CQPHC_MIN_DEC_FACTOR_S)
-
+#define IRDMA_CQPHC_MIN_DEC_FACTOR GENMASK_ULL(59, 56)
#define IRDMA_CQPHC_DCQCN_T_S 0
-#define IRDMA_CQPHC_DCQCN_T_M (0xffffULL << IRDMA_CQPHC_DCQCN_T_S)
-
+#define IRDMA_CQPHC_DCQCN_T GENMASK_ULL(15, 0)
#define IRDMA_CQPHC_HAI_FACTOR_S 32
-#define IRDMA_CQPHC_HAI_FACTOR_M \
- (0xffffULL << IRDMA_CQPHC_HAI_FACTOR_S)
-
+#define IRDMA_CQPHC_HAI_FACTOR GENMASK_ULL(47, 32)
#define IRDMA_CQPHC_RAI_FACTOR_S 48
-#define IRDMA_CQPHC_RAI_FACTOR_M \
- (0xffffULL << IRDMA_CQPHC_RAI_FACTOR_S)
-
+#define IRDMA_CQPHC_RAI_FACTOR GENMASK_ULL(63, 48)
#define IRDMA_CQPHC_DCQCN_B_S 0
-#define IRDMA_CQPHC_DCQCN_B_M (0x1ffffffULL << IRDMA_CQPHC_DCQCN_B_S)
-
+#define IRDMA_CQPHC_DCQCN_B GENMASK_ULL(24, 0)
#define IRDMA_CQPHC_DCQCN_F_S 25
-#define IRDMA_CQPHC_DCQCN_F_M (0x7ULL << IRDMA_CQPHC_DCQCN_F_S)
-
+#define IRDMA_CQPHC_DCQCN_F GENMASK_ULL(27, 25)
#define IRDMA_CQPHC_CC_CFG_VALID_S 31
-#define IRDMA_CQPHC_CC_CFG_VALID_M BIT_ULL(IRDMA_CQPHC_CC_CFG_VALID_S)
-
+#define IRDMA_CQPHC_CC_CFG_VALID BIT_ULL(31)
#define IRDMA_CQPHC_RREDUCE_MPERIOD_S 32
-#define IRDMA_CQPHC_RREDUCE_MPERIOD_M \
- (0xffffffffULL << IRDMA_CQPHC_RREDUCE_MPERIOD_S)
-
+#define IRDMA_CQPHC_RREDUCE_MPERIOD GENMASK_ULL(63, 32)
#define IRDMA_CQPHC_HW_MINVER_S 0
-#define IRDMA_CQPHC_HW_MINVER_M (0xffffULL << IRDMA_CQPHC_HW_MINVER_S)
+#define IRDMA_CQPHC_HW_MINVER GENMASK_ULL(15, 0)
#define IRDMA_CQPHC_HW_MAJVER_GEN_1 0
#define IRDMA_CQPHC_HW_MAJVER_GEN_2 1
#define IRDMA_CQPHC_HW_MAJVER_GEN_3 2
-
#define IRDMA_CQPHC_HW_MAJVER_S 16
-#define IRDMA_CQPHC_HW_MAJVER_M (0xffffULL << IRDMA_CQPHC_HW_MAJVER_S)
-
+#define IRDMA_CQPHC_HW_MAJVER GENMASK_ULL(31, 16)
#define IRDMA_CQPHC_CEQPERVF_S 32
-#define IRDMA_CQPHC_CEQPERVF_M (0xffULL << IRDMA_CQPHC_CEQPERVF_S)
+#define IRDMA_CQPHC_CEQPERVF GENMASK_ULL(39, 32)
#define IRDMA_CQPHC_EN_REM_ENDPOINT_TRK_S 3
-#define IRDMA_CQPHC_EN_REM_ENDPOINT_TRK_M BIT_ULL(IRDMA_CQPHC_EN_REM_ENDPOINT_TRK_S)
+#define IRDMA_CQPHC_EN_REM_ENDPOINT_TRK BIT_ULL(3)
#define IRDMA_CQPHC_ENABLED_VFS_S 32
-#define IRDMA_CQPHC_ENABLED_VFS_M (0x3fULL << IRDMA_CQPHC_ENABLED_VFS_S)
+#define IRDMA_CQPHC_ENABLED_VFS GENMASK_ULL(37, 32)
#define IRDMA_CQPHC_HMC_PROFILE_S 0
-#define IRDMA_CQPHC_HMC_PROFILE_M (0x7ULL << IRDMA_CQPHC_HMC_PROFILE_S)
-
+#define IRDMA_CQPHC_HMC_PROFILE GENMASK_ULL(2, 0)
#define IRDMA_CQPHC_SVER_S 24
-#define IRDMA_CQPHC_SVER_M (0xffULL << IRDMA_CQPHC_SVER_S)
-
+#define IRDMA_CQPHC_SVER GENMASK_ULL(31, 24)
#define IRDMA_CQPHC_SQBASE_S 9
-#define IRDMA_CQPHC_SQBASE_M \
- (0xfffffffffffffeULL << IRDMA_CQPHC_SQBASE_S)
+#define IRDMA_CQPHC_SQBASE GENMASK_ULL(63, 9)
#define IRDMA_CQPHC_QPCTX_S 0
-#define IRDMA_CQPHC_QPCTX_M \
- (0xffffffffffffffffULL << IRDMA_CQPHC_QPCTX_S)
-
-/* iWARP QP Doorbell shadow area */
+#define IRDMA_CQPHC_QPCTX GENMASK_ULL(63, 0)
#define IRDMA_QP_DBSA_HW_SQ_TAIL_S 0
-#define IRDMA_QP_DBSA_HW_SQ_TAIL_M \
- (0x7fffULL << IRDMA_QP_DBSA_HW_SQ_TAIL_S)
-
-/* Completion Queue Doorbell shadow area */
+#define IRDMA_QP_DBSA_HW_SQ_TAIL GENMASK_ULL(14, 0)
#define IRDMA_CQ_DBSA_CQEIDX_S 0
-#define IRDMA_CQ_DBSA_CQEIDX_M (0xfffffULL << IRDMA_CQ_DBSA_CQEIDX_S)
-
+#define IRDMA_CQ_DBSA_CQEIDX GENMASK_ULL(19, 0)
#define IRDMA_CQ_DBSA_SW_CQ_SELECT_S 0
-#define IRDMA_CQ_DBSA_SW_CQ_SELECT_M \
- (0x3fffULL << IRDMA_CQ_DBSA_SW_CQ_SELECT_S)
-
+#define IRDMA_CQ_DBSA_SW_CQ_SELECT GENMASK_ULL(13, 0)
#define IRDMA_CQ_DBSA_ARM_NEXT_S 14
-#define IRDMA_CQ_DBSA_ARM_NEXT_M BIT_ULL(IRDMA_CQ_DBSA_ARM_NEXT_S)
-
+#define IRDMA_CQ_DBSA_ARM_NEXT BIT_ULL(14)
#define IRDMA_CQ_DBSA_ARM_NEXT_SE_S 15
-#define IRDMA_CQ_DBSA_ARM_NEXT_SE_M BIT_ULL(IRDMA_CQ_DBSA_ARM_NEXT_SE_S)
-
+#define IRDMA_CQ_DBSA_ARM_NEXT_SE BIT_ULL(15)
#define IRDMA_CQ_DBSA_ARM_SEQ_NUM_S 16
-#define IRDMA_CQ_DBSA_ARM_SEQ_NUM_M \
- (0x3ULL << IRDMA_CQ_DBSA_ARM_SEQ_NUM_S)
+#define IRDMA_CQ_DBSA_ARM_SEQ_NUM GENMASK_ULL(17, 16)
/* CQP and iWARP Completion Queue */
#define IRDMA_CQ_QPCTX_S IRDMA_CQPHC_QPCTX_S
-#define IRDMA_CQ_QPCTX_M IRDMA_CQPHC_QPCTX_M
+#define IRDMA_CQ_QPCTX IRDMA_CQPHC_QPCTX
#define IRDMA_CCQ_OPRETVAL_S 0
-#define IRDMA_CCQ_OPRETVAL_M (0xffffffffULL << IRDMA_CCQ_OPRETVAL_S)
+#define IRDMA_CCQ_OPRETVAL GENMASK_ULL(31, 0)
#define IRDMA_CQ_MINERR_S 0
-#define IRDMA_CQ_MINERR_M (0xffffULL << IRDMA_CQ_MINERR_S)
-
+#define IRDMA_CQ_MINERR GENMASK_ULL(15, 0)
#define IRDMA_CQ_MAJERR_S 16
-#define IRDMA_CQ_MAJERR_M (0xffffULL << IRDMA_CQ_MAJERR_S)
-
+#define IRDMA_CQ_MAJERR GENMASK_ULL(31, 16)
#define IRDMA_CQ_WQEIDX_S 32
-#define IRDMA_CQ_WQEIDX_M (0x7fffULL << IRDMA_CQ_WQEIDX_S)
-
+#define IRDMA_CQ_WQEIDX GENMASK_ULL(46, 32)
#define IRDMA_CQ_EXTCQE_S 50
-#define IRDMA_CQ_EXTCQE_M BIT_ULL(IRDMA_CQ_EXTCQE_S)
-
+#define IRDMA_CQ_EXTCQE BIT_ULL(50)
#define IRDMA_OOO_CMPL_S 54
-#define IRDMA_OOO_CMPL_M BIT_ULL(IRDMA_OOO_CMPL_S)
-
+#define IRDMA_OOO_CMPL BIT_ULL(54)
#define IRDMA_CQ_ERROR_S 55
-#define IRDMA_CQ_ERROR_M BIT_ULL(IRDMA_CQ_ERROR_S)
-
+#define IRDMA_CQ_ERROR BIT_ULL(55)
#define IRDMA_CQ_SQ_S 62
-#define IRDMA_CQ_SQ_M BIT_ULL(IRDMA_CQ_SQ_S)
+#define IRDMA_CQ_SQ BIT_ULL(62)
#define IRDMA_CQ_VALID_S 63
-#define IRDMA_CQ_VALID_M BIT_ULL(IRDMA_CQ_VALID_S)
-
-#define IRDMA_CQ_IMMVALID_S 62
-#define IRDMA_CQ_IMMVALID_M BIT_ULL(IRDMA_CQ_IMMVALID_S)
-
+#define IRDMA_CQ_VALID BIT_ULL(63)
+#define IRDMA_CQ_IMMVALID BIT_ULL(62)
#define IRDMA_CQ_UDSMACVALID_S 61
-#define IRDMA_CQ_UDSMACVALID_M BIT_ULL(IRDMA_CQ_UDSMACVALID_S)
-
+#define IRDMA_CQ_UDSMACVALID BIT_ULL(61)
#define IRDMA_CQ_UDVLANVALID_S 60
-#define IRDMA_CQ_UDVLANVALID_M BIT_ULL(IRDMA_CQ_UDVLANVALID_S)
-
+#define IRDMA_CQ_UDVLANVALID BIT_ULL(60)
#define IRDMA_CQ_UDSMAC_S 0
-#define IRDMA_CQ_UDSMAC_M (0xffffffffffffULL << IRDMA_CQ_UDSMAC_S)
-
+#define IRDMA_CQ_UDSMAC GENMASK_ULL(47, 0)
#define IRDMA_CQ_UDVLAN_S 48
-#define IRDMA_CQ_UDVLAN_M (0xffffULL << IRDMA_CQ_UDVLAN_S)
+#define IRDMA_CQ_UDVLAN GENMASK_ULL(63, 48)
#define IRDMA_CQ_IMMDATA_S 0
-#define IRDMA_CQ_IMMDATA_M (0xffffffffffffffffULL << IRDMA_CQ_IMMVALID_S)
-
+#define IRDMA_CQ_IMMVALID_S 62
+#define IRDMA_CQ_IMMDATA GENMASK_ULL(125, 62)
#define IRDMA_CQ_IMMDATALOW32_S 0
-#define IRDMA_CQ_IMMDATALOW32_M (0xffffffffULL << IRDMA_CQ_IMMDATALOW32_S)
-
+#define IRDMA_CQ_IMMDATALOW32 GENMASK_ULL(31, 0)
#define IRDMA_CQ_IMMDATAUP32_S 32
-#define IRDMA_CQ_IMMDATAUP32_M (0xffffffffULL << IRDMA_CQ_IMMDATAUP32_S)
-
+#define IRDMA_CQ_IMMDATAUP32 GENMASK_ULL(63, 32)
#define IRDMACQ_PAYLDLEN_S 0
-#define IRDMACQ_PAYLDLEN_M (0xffffffffULL << IRDMACQ_PAYLDLEN_S)
-
-#define IRDMACQ_TCPSEQNUMRTT_S 32
-#define IRDMACQ_TCPSEQNUMRTT_M (0xffffffffULL << IRDMACQ_TCPSEQNUMRTT_S)
-
+#define IRDMACQ_PAYLDLEN GENMASK_ULL(31, 0)
+#define IRDMACQ_TCPSQN_ROCEPSN_RTT_TS_S 32
+#define IRDMACQ_TCPSQN_ROCEPSN_RTT_TS GENMASK_ULL(63, 32)
#define IRDMACQ_INVSTAG_S 0
-#define IRDMACQ_INVSTAG_M (0xffffffffULL << IRDMACQ_INVSTAG_S)
-
+#define IRDMACQ_INVSTAG GENMASK_ULL(31, 0)
#define IRDMACQ_QPID_S 32
-#define IRDMACQ_QPID_M (0xffffffULL << IRDMACQ_QPID_S)
+#define IRDMACQ_QPID GENMASK_ULL(55, 32)
#define IRDMACQ_UDSRCQPN_S 0
-#define IRDMACQ_UDSRCQPN_M (0xffffffffULL << IRDMACQ_UDSRCQPN_S)
-
+#define IRDMACQ_UDSRCQPN GENMASK_ULL(31, 0)
#define IRDMACQ_PSHDROP_S 51
-#define IRDMACQ_PSHDROP_M BIT_ULL(IRDMACQ_PSHDROP_S)
-
+#define IRDMACQ_PSHDROP BIT_ULL(51)
#define IRDMACQ_STAG_S 53
-#define IRDMACQ_STAG_M BIT_ULL(IRDMACQ_STAG_S)
-
+#define IRDMACQ_STAG BIT_ULL(53)
#define IRDMACQ_IPV4_S 53
-#define IRDMACQ_IPV4_M BIT_ULL(IRDMACQ_IPV4_S)
-
+#define IRDMACQ_IPV4 BIT_ULL(53)
#define IRDMACQ_SOEVENT_S 54
-#define IRDMACQ_SOEVENT_M BIT_ULL(IRDMACQ_SOEVENT_S)
-
+#define IRDMACQ_SOEVENT BIT_ULL(54)
#define IRDMACQ_OP_S 56
-#define IRDMACQ_OP_M (0x3fULL << IRDMACQ_OP_S)
+#define IRDMACQ_OP GENMASK_ULL(61, 56)
-/* CEQE format */
#define IRDMA_CEQE_CQCTX_S 0
-#define IRDMA_CEQE_CQCTX_M \
- (0x7fffffffffffffffULL << IRDMA_CEQE_CQCTX_S)
-
+#define IRDMA_CEQE_CQCTX GENMASK_ULL(62, 0)
#define IRDMA_CEQE_VALID_S 63
-#define IRDMA_CEQE_VALID_M BIT_ULL(IRDMA_CEQE_VALID_S)
+#define IRDMA_CEQE_VALID BIT_ULL(63)
/* AEQE format */
#define IRDMA_AEQE_COMPCTX_S IRDMA_CQPHC_QPCTX_S
-#define IRDMA_AEQE_COMPCTX_M IRDMA_CQPHC_QPCTX_M
-
+#define IRDMA_AEQE_COMPCTX IRDMA_CQPHC_QPCTX
#define IRDMA_AEQE_QPCQID_LOW_S 0
-#define IRDMA_AEQE_QPCQID_LOW_M (0x3ffffULL << IRDMA_AEQE_QPCQID_LOW_S)
-
+#define IRDMA_AEQE_QPCQID_LOW GENMASK_ULL(17, 0)
#define IRDMA_AEQE_QPCQID_HI_S 46
-#define IRDMA_AEQE_QPCQID_HI_M BIT_ULL(IRDMA_AEQE_QPCQID_HI_S)
-
+#define IRDMA_AEQE_QPCQID_HI BIT_ULL(46)
#define IRDMA_AEQE_WQDESCIDX_S 18
-#define IRDMA_AEQE_WQDESCIDX_M (0x7fffULL << IRDMA_AEQE_WQDESCIDX_S)
-
+#define IRDMA_AEQE_WQDESCIDX GENMASK_ULL(32, 18)
#define IRDMA_AEQE_OVERFLOW_S 33
-#define IRDMA_AEQE_OVERFLOW_M BIT_ULL(IRDMA_AEQE_OVERFLOW_S)
-
+#define IRDMA_AEQE_OVERFLOW BIT_ULL(33)
#define IRDMA_AEQE_AECODE_S 34
-#define IRDMA_AEQE_AECODE_M (0xfffULL << IRDMA_AEQE_AECODE_S)
-
+#define IRDMA_AEQE_AECODE GENMASK_ULL(45, 34)
#define IRDMA_AEQE_AESRC_S 50
-#define IRDMA_AEQE_AESRC_M (0xfULL << IRDMA_AEQE_AESRC_S)
-
+#define IRDMA_AEQE_AESRC GENMASK_ULL(53, 50)
#define IRDMA_AEQE_IWSTATE_S 54
-#define IRDMA_AEQE_IWSTATE_M (0x7ULL << IRDMA_AEQE_IWSTATE_S)
-
+#define IRDMA_AEQE_IWSTATE GENMASK_ULL(56, 54)
#define IRDMA_AEQE_TCPSTATE_S 57
-#define IRDMA_AEQE_TCPSTATE_M (0xfULL << IRDMA_AEQE_TCPSTATE_S)
-
+#define IRDMA_AEQE_TCPSTATE GENMASK_ULL(60, 57)
#define IRDMA_AEQE_Q2DATA_S 61
-#define IRDMA_AEQE_Q2DATA_M (0x3ULL << IRDMA_AEQE_Q2DATA_S)
-
+#define IRDMA_AEQE_Q2DATA GENMASK_ULL(62, 61)
#define IRDMA_AEQE_VALID_S 63
-#define IRDMA_AEQE_VALID_M BIT_ULL(IRDMA_AEQE_VALID_S)
+#define IRDMA_AEQE_VALID BIT_ULL(63)
#define IRDMA_UDA_QPSQ_NEXT_HDR_S 16
-#define IRDMA_UDA_QPSQ_NEXT_HDR_M ((u64)0xff << IRDMA_UDA_QPSQ_NEXT_HDR_S)
-
+#define IRDMA_UDA_QPSQ_NEXT_HDR GENMASK_ULL(23, 16)
#define IRDMA_UDA_QPSQ_OPCODE_S 32
-#define IRDMA_UDA_QPSQ_OPCODE_M ((u64)0x3f << IRDMA_UDA_QPSQ_OPCODE_S)
-
+#define IRDMA_UDA_QPSQ_OPCODE GENMASK_ULL(37, 32)
#define IRDMA_UDA_QPSQ_L4LEN_S 42
-#define IRDMA_UDA_QPSQ_L4LEN_M ((u64)0xf << IRDMA_UDA_QPSQ_L4LEN_S)
-
+#define IRDMA_UDA_QPSQ_L4LEN GENMASK_ULL(45, 42)
#define IRDMA_GEN1_UDA_QPSQ_L4LEN_S 24
-#define IRDMA_GEN1_UDA_QPSQ_L4LEN_M ((u64)0xf << IRDMA_GEN1_UDA_QPSQ_L4LEN_S)
-
+#define IRDMA_GEN1_UDA_QPSQ_L4LEN GENMASK_ULL(27, 24)
#define IRDMA_UDA_QPSQ_AHIDX_S 0
-#define IRDMA_UDA_QPSQ_AHIDX_M ((u64)0x1ffff << IRDMA_UDA_QPSQ_AHIDX_S)
-
+#define IRDMA_UDA_QPSQ_AHIDX GENMASK_ULL(16, 0)
#define IRDMA_UDA_QPSQ_VALID_S 63
-#define IRDMA_UDA_QPSQ_VALID_M \
- BIT_ULL(IRDMA_UDA_QPSQ_VALID_S)
-
+#define IRDMA_UDA_QPSQ_VALID BIT_ULL(63)
#define IRDMA_UDA_QPSQ_SIGCOMPL_S 62
-#define IRDMA_UDA_QPSQ_SIGCOMPL_M BIT_ULL(IRDMA_UDA_QPSQ_SIGCOMPL_S)
-
+#define IRDMA_UDA_QPSQ_SIGCOMPL BIT_ULL(62)
#define IRDMA_UDA_QPSQ_MACLEN_S 56
-#define IRDMA_UDA_QPSQ_MACLEN_M \
- ((u64)0x7f << IRDMA_UDA_QPSQ_MACLEN_S)
-
+#define IRDMA_UDA_QPSQ_MACLEN GENMASK_ULL(62, 56)
#define IRDMA_UDA_QPSQ_IPLEN_S 48
-#define IRDMA_UDA_QPSQ_IPLEN_M \
- ((u64)0x7f << IRDMA_UDA_QPSQ_IPLEN_S)
-
+#define IRDMA_UDA_QPSQ_IPLEN GENMASK_ULL(54, 48)
#define IRDMA_UDA_QPSQ_L4T_S 30
-#define IRDMA_UDA_QPSQ_L4T_M \
- ((u64)0x3 << IRDMA_UDA_QPSQ_L4T_S)
-
+#define IRDMA_UDA_QPSQ_L4T GENMASK_ULL(31, 30)
#define IRDMA_UDA_QPSQ_IIPT_S 28
-#define IRDMA_UDA_QPSQ_IIPT_M \
- ((u64)0x3 << IRDMA_UDA_QPSQ_IIPT_S)
-
+#define IRDMA_UDA_QPSQ_IIPT GENMASK_ULL(29, 28)
#define IRDMA_UDA_PAYLOADLEN_S 0
-#define IRDMA_UDA_PAYLOADLEN_M ((u64)0x3fff << IRDMA_UDA_PAYLOADLEN_S)
-
+#define IRDMA_UDA_PAYLOADLEN GENMASK_ULL(13, 0)
#define IRDMA_UDA_HDRLEN_S 16
-#define IRDMA_UDA_HDRLEN_M ((u64)0x1ff << IRDMA_UDA_HDRLEN_S)
-
+#define IRDMA_UDA_HDRLEN GENMASK_ULL(24, 16)
#define IRDMA_VLAN_TAG_VALID_S 50
-#define IRDMA_VLAN_TAG_VALID_M BIT_ULL(IRDMA_VLAN_TAG_VALID_S)
-
+#define IRDMA_VLAN_TAG_VALID BIT_ULL(50)
#define IRDMA_UDA_L3PROTO_S 0
-#define IRDMA_UDA_L3PROTO_M ((u64)0x3 << IRDMA_UDA_L3PROTO_S)
-
+#define IRDMA_UDA_L3PROTO GENMASK_ULL(1, 0)
#define IRDMA_UDA_L4PROTO_S 16
-#define IRDMA_UDA_L4PROTO_M ((u64)0x3 << IRDMA_UDA_L4PROTO_S)
-
+#define IRDMA_UDA_L4PROTO GENMASK_ULL(17, 16)
#define IRDMA_UDA_QPSQ_DOLOOPBACK_S 44
-#define IRDMA_UDA_QPSQ_DOLOOPBACK_M \
- BIT_ULL(IRDMA_UDA_QPSQ_DOLOOPBACK_S)
-
-/* CQP SQ WQE common fields */
+#define IRDMA_UDA_QPSQ_DOLOOPBACK BIT_ULL(44)
#define IRDMA_CQPSQ_BUFSIZE_S 0
-#define IRDMA_CQPSQ_BUFSIZE_M (0xffffffffULL << IRDMA_CQPSQ_BUFSIZE_S)
-
+#define IRDMA_CQPSQ_BUFSIZE GENMASK_ULL(31, 0)
#define IRDMA_CQPSQ_OPCODE_S 32
-#define IRDMA_CQPSQ_OPCODE_M (0x3fULL << IRDMA_CQPSQ_OPCODE_S)
-
+#define IRDMA_CQPSQ_OPCODE GENMASK_ULL(37, 32)
#define IRDMA_CQPSQ_WQEVALID_S 63
-#define IRDMA_CQPSQ_WQEVALID_M BIT_ULL(IRDMA_CQPSQ_WQEVALID_S)
-
+#define IRDMA_CQPSQ_WQEVALID BIT_ULL(63)
#define IRDMA_CQPSQ_TPHVAL_S 0
-#define IRDMA_CQPSQ_TPHVAL_M (0xffULL << IRDMA_CQPSQ_TPHVAL_S)
+#define IRDMA_CQPSQ_TPHVAL GENMASK_ULL(7, 0)
#define IRDMA_CQPSQ_VSIIDX_S 8
-#define IRDMA_CQPSQ_VSIIDX_M (0x3ffULL << IRDMA_CQPSQ_VSIIDX_S)
-
+#define IRDMA_CQPSQ_VSIIDX GENMASK_ULL(17, 8)
#define IRDMA_CQPSQ_TPHEN_S 60
-#define IRDMA_CQPSQ_TPHEN_M BIT_ULL(IRDMA_CQPSQ_TPHEN_S)
+#define IRDMA_CQPSQ_TPHEN BIT_ULL(60)
#define IRDMA_CQPSQ_PBUFADDR_S IRDMA_CQPHC_QPCTX_S
-#define IRDMA_CQPSQ_PBUFADDR_M IRDMA_CQPHC_QPCTX_M
+#define IRDMA_CQPSQ_PBUFADDR IRDMA_CQPHC_QPCTX
/* Create/Modify/Destroy QP */
#define IRDMA_CQPSQ_QP_NEWMSS_S 32
-#define IRDMA_CQPSQ_QP_NEWMSS_M (0x3fffULL << IRDMA_CQPSQ_QP_NEWMSS_S)
-
+#define IRDMA_CQPSQ_QP_NEWMSS GENMASK_ULL(45, 32)
#define IRDMA_CQPSQ_QP_TERMLEN_S 48
-#define IRDMA_CQPSQ_QP_TERMLEN_M (0xfULL << IRDMA_CQPSQ_QP_TERMLEN_S)
+#define IRDMA_CQPSQ_QP_TERMLEN GENMASK_ULL(51, 48)
#define IRDMA_CQPSQ_QP_QPCTX_S IRDMA_CQPHC_QPCTX_S
-#define IRDMA_CQPSQ_QP_QPCTX_M IRDMA_CQPHC_QPCTX_M
+#define IRDMA_CQPSQ_QP_QPCTX IRDMA_CQPHC_QPCTX
#define IRDMA_CQPSQ_QP_QPID_S 0
#define IRDMA_CQPSQ_QP_QPID_M (0xFFFFFFUL)
#define IRDMA_CQPSQ_QP_OP_S 32
#define IRDMA_CQPSQ_QP_OP_M IRDMACQ_OP_M
-
#define IRDMA_CQPSQ_QP_ORDVALID_S 42
-#define IRDMA_CQPSQ_QP_ORDVALID_M BIT_ULL(IRDMA_CQPSQ_QP_ORDVALID_S)
-
+#define IRDMA_CQPSQ_QP_ORDVALID BIT_ULL(42)
#define IRDMA_CQPSQ_QP_TOECTXVALID_S 43
-#define IRDMA_CQPSQ_QP_TOECTXVALID_M \
- BIT_ULL(IRDMA_CQPSQ_QP_TOECTXVALID_S)
-
+#define IRDMA_CQPSQ_QP_TOECTXVALID BIT_ULL(43)
#define IRDMA_CQPSQ_QP_CACHEDVARVALID_S 44
-#define IRDMA_CQPSQ_QP_CACHEDVARVALID_M \
- BIT_ULL(IRDMA_CQPSQ_QP_CACHEDVARVALID_S)
-
+#define IRDMA_CQPSQ_QP_CACHEDVARVALID BIT_ULL(44)
#define IRDMA_CQPSQ_QP_VQ_S 45
-#define IRDMA_CQPSQ_QP_VQ_M BIT_ULL(IRDMA_CQPSQ_QP_VQ_S)
-
+#define IRDMA_CQPSQ_QP_VQ BIT_ULL(45)
#define IRDMA_CQPSQ_QP_FORCELOOPBACK_S 46
-#define IRDMA_CQPSQ_QP_FORCELOOPBACK_M \
- BIT_ULL(IRDMA_CQPSQ_QP_FORCELOOPBACK_S)
-
+#define IRDMA_CQPSQ_QP_FORCELOOPBACK BIT_ULL(46)
#define IRDMA_CQPSQ_QP_CQNUMVALID_S 47
-#define IRDMA_CQPSQ_QP_CQNUMVALID_M \
- BIT_ULL(IRDMA_CQPSQ_QP_CQNUMVALID_S)
-
+#define IRDMA_CQPSQ_QP_CQNUMVALID BIT_ULL(47)
#define IRDMA_CQPSQ_QP_QPTYPE_S 48
-#define IRDMA_CQPSQ_QP_QPTYPE_M (0x7ULL << IRDMA_CQPSQ_QP_QPTYPE_S)
-
+#define IRDMA_CQPSQ_QP_QPTYPE GENMASK_ULL(50, 48)
#define IRDMA_CQPSQ_QP_MACVALID_S 51
-#define IRDMA_CQPSQ_QP_MACVALID_M BIT_ULL(IRDMA_CQPSQ_QP_MACVALID_S)
-
+#define IRDMA_CQPSQ_QP_MACVALID BIT_ULL(51)
#define IRDMA_CQPSQ_QP_MSSCHANGE_S 52
-#define IRDMA_CQPSQ_QP_MSSCHANGE_M BIT_ULL(IRDMA_CQPSQ_QP_MSSCHANGE_S)
+#define IRDMA_CQPSQ_QP_MSSCHANGE BIT_ULL(52)
#define IRDMA_CQPSQ_QP_IGNOREMWBOUND_S 54
-#define IRDMA_CQPSQ_QP_IGNOREMWBOUND_M \
- BIT_ULL(IRDMA_CQPSQ_QP_IGNOREMWBOUND_S)
-
+#define IRDMA_CQPSQ_QP_IGNOREMWBOUND BIT_ULL(54)
#define IRDMA_CQPSQ_QP_REMOVEHASHENTRY_S 55
-#define IRDMA_CQPSQ_QP_REMOVEHASHENTRY_M \
- BIT_ULL(IRDMA_CQPSQ_QP_REMOVEHASHENTRY_S)
-
+#define IRDMA_CQPSQ_QP_REMOVEHASHENTRY BIT_ULL(55)
#define IRDMA_CQPSQ_QP_TERMACT_S 56
-#define IRDMA_CQPSQ_QP_TERMACT_M (0x3ULL << IRDMA_CQPSQ_QP_TERMACT_S)
-
+#define IRDMA_CQPSQ_QP_TERMACT GENMASK_ULL(57, 56)
#define IRDMA_CQPSQ_QP_RESETCON_S 58
-#define IRDMA_CQPSQ_QP_RESETCON_M BIT_ULL(IRDMA_CQPSQ_QP_RESETCON_S)
-
+#define IRDMA_CQPSQ_QP_RESETCON BIT_ULL(58)
#define IRDMA_CQPSQ_QP_ARPTABIDXVALID_S 59
-#define IRDMA_CQPSQ_QP_ARPTABIDXVALID_M \
- BIT_ULL(IRDMA_CQPSQ_QP_ARPTABIDXVALID_S)
-
+#define IRDMA_CQPSQ_QP_ARPTABIDXVALID BIT_ULL(59)
#define IRDMA_CQPSQ_QP_NEXTIWSTATE_S 60
-#define IRDMA_CQPSQ_QP_NEXTIWSTATE_M \
- (0x7ULL << IRDMA_CQPSQ_QP_NEXTIWSTATE_S)
+#define IRDMA_CQPSQ_QP_NEXTIWSTATE GENMASK_ULL(62, 60)
#define IRDMA_CQPSQ_QP_DBSHADOWADDR_S IRDMA_CQPHC_QPCTX_S
-#define IRDMA_CQPSQ_QP_DBSHADOWADDR_M IRDMA_CQPHC_QPCTX_M
+#define IRDMA_CQPSQ_QP_DBSHADOWADDR IRDMA_CQPHC_QPCTX
-/* Create/Modify/Destroy CQ */
#define IRDMA_CQPSQ_CQ_CQSIZE_S 0
-#define IRDMA_CQPSQ_CQ_CQSIZE_M (0x1fffffULL << IRDMA_CQPSQ_CQ_CQSIZE_S)
-
+#define IRDMA_CQPSQ_CQ_CQSIZE GENMASK_ULL(20, 0)
#define IRDMA_CQPSQ_CQ_CQCTX_S 0
-#define IRDMA_CQPSQ_CQ_CQCTX_M \
- (0x7fffffffffffffffULL << IRDMA_CQPSQ_CQ_CQCTX_S)
-
+#define IRDMA_CQPSQ_CQ_CQCTX GENMASK_ULL(62, 0)
#define IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD_S 0
-#define IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD_M \
- (0x3ffff << IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD_S)
+#define IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD GENMASK(17, 0)
#define IRDMA_CQPSQ_CQ_OP_S 32
-#define IRDMA_CQPSQ_CQ_OP_M (0x3fULL << IRDMA_CQPSQ_CQ_OP_S)
-
+#define IRDMA_CQPSQ_CQ_OP GENMASK_ULL(37, 32)
#define IRDMA_CQPSQ_CQ_CQRESIZE_S 43
-#define IRDMA_CQPSQ_CQ_CQRESIZE_M BIT_ULL(IRDMA_CQPSQ_CQ_CQRESIZE_S)
-
+#define IRDMA_CQPSQ_CQ_CQRESIZE BIT_ULL(43)
#define IRDMA_CQPSQ_CQ_LPBLSIZE_S 44
-#define IRDMA_CQPSQ_CQ_LPBLSIZE_M (3ULL << IRDMA_CQPSQ_CQ_LPBLSIZE_S)
-
+#define IRDMA_CQPSQ_CQ_LPBLSIZE GENMASK_ULL(45, 44)
#define IRDMA_CQPSQ_CQ_CHKOVERFLOW_S 46
-#define IRDMA_CQPSQ_CQ_CHKOVERFLOW_M \
- BIT_ULL(IRDMA_CQPSQ_CQ_CHKOVERFLOW_S)
-
+#define IRDMA_CQPSQ_CQ_CHKOVERFLOW BIT_ULL(46)
#define IRDMA_CQPSQ_CQ_VIRTMAP_S 47
-#define IRDMA_CQPSQ_CQ_VIRTMAP_M BIT_ULL(IRDMA_CQPSQ_CQ_VIRTMAP_S)
-
+#define IRDMA_CQPSQ_CQ_VIRTMAP BIT_ULL(47)
#define IRDMA_CQPSQ_CQ_ENCEQEMASK_S 48
-#define IRDMA_CQPSQ_CQ_ENCEQEMASK_M \
- BIT_ULL(IRDMA_CQPSQ_CQ_ENCEQEMASK_S)
-
+#define IRDMA_CQPSQ_CQ_ENCEQEMASK BIT_ULL(48)
#define IRDMA_CQPSQ_CQ_CEQIDVALID_S 49
-#define IRDMA_CQPSQ_CQ_CEQIDVALID_M \
- BIT_ULL(IRDMA_CQPSQ_CQ_CEQIDVALID_S)
-
+#define IRDMA_CQPSQ_CQ_CEQIDVALID BIT_ULL(49)
#define IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT_S 61
-#define IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT_M \
- BIT_ULL(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT_S)
-
+#define IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT BIT_ULL(61)
#define IRDMA_CQPSQ_CQ_FIRSTPMPBLIDX_S 0
-#define IRDMA_CQPSQ_CQ_FIRSTPMPBLIDX_M \
- (0xfffffffULL << IRDMA_CQPSQ_CQ_FIRSTPMPBLIDX_S)
+#define IRDMA_CQPSQ_CQ_FIRSTPMPBLIDX GENMASK_ULL(27, 0)
/* Allocate/Register/Register Shared/Deallocate Stag */
#define IRDMA_CQPSQ_STAG_VA_FBO_S IRDMA_CQPHC_QPCTX_S
-#define IRDMA_CQPSQ_STAG_VA_FBO_M IRDMA_CQPHC_QPCTX_M
-
+#define IRDMA_CQPSQ_STAG_VA_FBO IRDMA_CQPHC_QPCTX
#define IRDMA_CQPSQ_STAG_STAGLEN_S 0
-#define IRDMA_CQPSQ_STAG_STAGLEN_M \
- (0x3fffffffffffULL << IRDMA_CQPSQ_STAG_STAGLEN_S)
-
+#define IRDMA_CQPSQ_STAG_STAGLEN GENMASK_ULL(45, 0)
#define IRDMA_CQPSQ_STAG_KEY_S 0
-#define IRDMA_CQPSQ_STAG_KEY_M (0xffULL << IRDMA_CQPSQ_STAG_KEY_S)
-
+#define IRDMA_CQPSQ_STAG_KEY GENMASK_ULL(7, 0)
#define IRDMA_CQPSQ_STAG_IDX_S 8
-#define IRDMA_CQPSQ_STAG_IDX_M (0xffffffULL << IRDMA_CQPSQ_STAG_IDX_S)
-
+#define IRDMA_CQPSQ_STAG_IDX GENMASK_ULL(31, 8)
#define IRDMA_CQPSQ_STAG_PARENTSTAGIDX_S 32
-#define IRDMA_CQPSQ_STAG_PARENTSTAGIDX_M \
- (0xffffffULL << IRDMA_CQPSQ_STAG_PARENTSTAGIDX_S)
-
+#define IRDMA_CQPSQ_STAG_PARENTSTAGIDX GENMASK_ULL(55, 32)
#define IRDMA_CQPSQ_STAG_MR_S 43
-#define IRDMA_CQPSQ_STAG_MR_M BIT_ULL(IRDMA_CQPSQ_STAG_MR_S)
-
+#define IRDMA_CQPSQ_STAG_MR BIT_ULL(43)
#define IRDMA_CQPSQ_STAG_MWTYPE_S 42
-#define IRDMA_CQPSQ_STAG_MWTYPE_M BIT_ULL(IRDMA_CQPSQ_STAG_MWTYPE_S)
-
+#define IRDMA_CQPSQ_STAG_MWTYPE BIT_ULL(42)
#define IRDMA_CQPSQ_STAG_MW1_BIND_DONT_VLDT_KEY_S 58
-#define IRDMA_CQPSQ_STAG_MW1_BIND_DONT_VLDT_KEY_M \
- BIT_ULL(IRDMA_CQPSQ_STAG_MW1_BIND_DONT_VLDT_KEY_S)
+#define IRDMA_CQPSQ_STAG_MW1_BIND_DONT_VLDT_KEY BIT_ULL(58)
#define IRDMA_CQPSQ_STAG_LPBLSIZE_S IRDMA_CQPSQ_CQ_LPBLSIZE_S
#define IRDMA_CQPSQ_STAG_LPBLSIZE_M IRDMA_CQPSQ_CQ_LPBLSIZE_M
-
+#define IRDMA_CQPSQ_STAG_LPBLSIZE IRDMA_CQPSQ_CQ_LPBLSIZE
#define IRDMA_CQPSQ_STAG_HPAGESIZE_S 46
-#define IRDMA_CQPSQ_STAG_HPAGESIZE_M \
- ((u64)3 << IRDMA_CQPSQ_STAG_HPAGESIZE_S)
-
+#define IRDMA_CQPSQ_STAG_HPAGESIZE GENMASK_ULL(47, 46)
#define IRDMA_CQPSQ_STAG_ARIGHTS_S 48
-#define IRDMA_CQPSQ_STAG_ARIGHTS_M \
- (0x1fULL << IRDMA_CQPSQ_STAG_ARIGHTS_S)
-
+#define IRDMA_CQPSQ_STAG_ARIGHTS GENMASK_ULL(52, 48)
#define IRDMA_CQPSQ_STAG_REMACCENABLED_S 53
-#define IRDMA_CQPSQ_STAG_REMACCENABLED_M \
- BIT_ULL(IRDMA_CQPSQ_STAG_REMACCENABLED_S)
-
+#define IRDMA_CQPSQ_STAG_REMACCENABLED BIT_ULL(53)
#define IRDMA_CQPSQ_STAG_VABASEDTO_S 59
-#define IRDMA_CQPSQ_STAG_VABASEDTO_M \
- BIT_ULL(IRDMA_CQPSQ_STAG_VABASEDTO_S)
-
+#define IRDMA_CQPSQ_STAG_VABASEDTO BIT_ULL(59)
#define IRDMA_CQPSQ_STAG_USEHMCFNIDX_S 60
-#define IRDMA_CQPSQ_STAG_USEHMCFNIDX_M \
- BIT_ULL(IRDMA_CQPSQ_STAG_USEHMCFNIDX_S)
-
+#define IRDMA_CQPSQ_STAG_USEHMCFNIDX BIT_ULL(60)
#define IRDMA_CQPSQ_STAG_USEPFRID_S 61
-#define IRDMA_CQPSQ_STAG_USEPFRID_M \
- BIT_ULL(IRDMA_CQPSQ_STAG_USEPFRID_S)
+#define IRDMA_CQPSQ_STAG_USEPFRID BIT_ULL(61)
#define IRDMA_CQPSQ_STAG_PBA_S IRDMA_CQPHC_QPCTX_S
-#define IRDMA_CQPSQ_STAG_PBA_M IRDMA_CQPHC_QPCTX_M
-
+#define IRDMA_CQPSQ_STAG_PBA IRDMA_CQPHC_QPCTX
#define IRDMA_CQPSQ_STAG_HMCFNIDX_S 0
-#define IRDMA_CQPSQ_STAG_HMCFNIDX_M \
- (0x3fULL << IRDMA_CQPSQ_STAG_HMCFNIDX_S)
+#define IRDMA_CQPSQ_STAG_HMCFNIDX GENMASK_ULL(5, 0)
#define IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX_S 0
-#define IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX_M \
- (0xfffffffULL << IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX_S)
+#define IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX GENMASK_ULL(27, 0)
#define IRDMA_CQPSQ_QUERYSTAG_IDX_S IRDMA_CQPSQ_STAG_IDX_S
-#define IRDMA_CQPSQ_QUERYSTAG_IDX_M IRDMA_CQPSQ_STAG_IDX_M
-
-/* Manage Local MAC Table - MLM */
+#define IRDMA_CQPSQ_QUERYSTAG_IDX IRDMA_CQPSQ_STAG_IDX
#define IRDMA_CQPSQ_MLM_TABLEIDX_S 0
-#define IRDMA_CQPSQ_MLM_TABLEIDX_M \
- (0x3fULL << IRDMA_CQPSQ_MLM_TABLEIDX_S)
-
+#define IRDMA_CQPSQ_MLM_TABLEIDX GENMASK_ULL(5, 0)
#define IRDMA_CQPSQ_MLM_FREEENTRY_S 62
-#define IRDMA_CQPSQ_MLM_FREEENTRY_M \
- BIT_ULL(IRDMA_CQPSQ_MLM_FREEENTRY_S)
-
+#define IRDMA_CQPSQ_MLM_FREEENTRY BIT_ULL(62)
#define IRDMA_CQPSQ_MLM_IGNORE_REF_CNT_S 61
-#define IRDMA_CQPSQ_MLM_IGNORE_REF_CNT_M \
- BIT_ULL(IRDMA_CQPSQ_MLM_IGNORE_REF_CNT_S)
-
+#define IRDMA_CQPSQ_MLM_IGNORE_REF_CNT BIT_ULL(61)
#define IRDMA_CQPSQ_MLM_MAC0_S 0
-#define IRDMA_CQPSQ_MLM_MAC0_M (0xffULL << IRDMA_CQPSQ_MLM_MAC0_S)
-
+#define IRDMA_CQPSQ_MLM_MAC0 GENMASK_ULL(7, 0)
#define IRDMA_CQPSQ_MLM_MAC1_S 8
-#define IRDMA_CQPSQ_MLM_MAC1_M (0xffULL << IRDMA_CQPSQ_MLM_MAC1_S)
-
+#define IRDMA_CQPSQ_MLM_MAC1 GENMASK_ULL(15, 8)
#define IRDMA_CQPSQ_MLM_MAC2_S 16
-#define IRDMA_CQPSQ_MLM_MAC2_M (0xffULL << IRDMA_CQPSQ_MLM_MAC2_S)
-
+#define IRDMA_CQPSQ_MLM_MAC2 GENMASK_ULL(23, 16)
#define IRDMA_CQPSQ_MLM_MAC3_S 24
-#define IRDMA_CQPSQ_MLM_MAC3_M (0xffULL << IRDMA_CQPSQ_MLM_MAC3_S)
-
+#define IRDMA_CQPSQ_MLM_MAC3 GENMASK_ULL(31, 24)
#define IRDMA_CQPSQ_MLM_MAC4_S 32
-#define IRDMA_CQPSQ_MLM_MAC4_M (0xffULL << IRDMA_CQPSQ_MLM_MAC4_S)
-
+#define IRDMA_CQPSQ_MLM_MAC4 GENMASK_ULL(39, 32)
#define IRDMA_CQPSQ_MLM_MAC5_S 40
-#define IRDMA_CQPSQ_MLM_MAC5_M (0xffULL << IRDMA_CQPSQ_MLM_MAC5_S)
-
-/* Manage ARP Table - MAT */
+#define IRDMA_CQPSQ_MLM_MAC5 GENMASK_ULL(47, 40)
#define IRDMA_CQPSQ_MAT_REACHMAX_S 0
-#define IRDMA_CQPSQ_MAT_REACHMAX_M \
- (0xffffffffULL << IRDMA_CQPSQ_MAT_REACHMAX_S)
-
+#define IRDMA_CQPSQ_MAT_REACHMAX GENMASK_ULL(31, 0)
#define IRDMA_CQPSQ_MAT_MACADDR_S 0
-#define IRDMA_CQPSQ_MAT_MACADDR_M \
- (0xffffffffffffULL << IRDMA_CQPSQ_MAT_MACADDR_S)
-
+#define IRDMA_CQPSQ_MAT_MACADDR GENMASK_ULL(47, 0)
#define IRDMA_CQPSQ_MAT_ARPENTRYIDX_S 0
-#define IRDMA_CQPSQ_MAT_ARPENTRYIDX_M \
- (0xfffULL << IRDMA_CQPSQ_MAT_ARPENTRYIDX_S)
-
+#define IRDMA_CQPSQ_MAT_ARPENTRYIDX GENMASK_ULL(11, 0)
#define IRDMA_CQPSQ_MAT_ENTRYVALID_S 42
-#define IRDMA_CQPSQ_MAT_ENTRYVALID_M \
- BIT_ULL(IRDMA_CQPSQ_MAT_ENTRYVALID_S)
-
+#define IRDMA_CQPSQ_MAT_ENTRYVALID BIT_ULL(42)
#define IRDMA_CQPSQ_MAT_PERMANENT_S 43
-#define IRDMA_CQPSQ_MAT_PERMANENT_M \
- BIT_ULL(IRDMA_CQPSQ_MAT_PERMANENT_S)
-
+#define IRDMA_CQPSQ_MAT_PERMANENT BIT_ULL(43)
#define IRDMA_CQPSQ_MAT_QUERY_S 44
-#define IRDMA_CQPSQ_MAT_QUERY_M BIT_ULL(IRDMA_CQPSQ_MAT_QUERY_S)
-
-/* Manage VF PBLE Backing Pages - MVPBP*/
+#define IRDMA_CQPSQ_MAT_QUERY BIT_ULL(44)
#define IRDMA_CQPSQ_MVPBP_PD_ENTRY_CNT_S 0
-#define IRDMA_CQPSQ_MVPBP_PD_ENTRY_CNT_M \
- (0x3ffULL << IRDMA_CQPSQ_MVPBP_PD_ENTRY_CNT_S)
-
+#define IRDMA_CQPSQ_MVPBP_PD_ENTRY_CNT GENMASK_ULL(9, 0)
#define IRDMA_CQPSQ_MVPBP_FIRST_PD_INX_S 16
-#define IRDMA_CQPSQ_MVPBP_FIRST_PD_INX_M \
- (0x1ffULL << IRDMA_CQPSQ_MVPBP_FIRST_PD_INX_S)
-
+#define IRDMA_CQPSQ_MVPBP_FIRST_PD_INX GENMASK_ULL(24, 16)
#define IRDMA_CQPSQ_MVPBP_SD_INX_S 32
-#define IRDMA_CQPSQ_MVPBP_SD_INX_M \
- (0xfffULL << IRDMA_CQPSQ_MVPBP_SD_INX_S)
-
+#define IRDMA_CQPSQ_MVPBP_SD_INX GENMASK_ULL(43, 32)
#define IRDMA_CQPSQ_MVPBP_INV_PD_ENT_S 62
-#define IRDMA_CQPSQ_MVPBP_INV_PD_ENT_M \
- BIT_ULL(IRDMA_CQPSQ_MVPBP_INV_PD_ENT_S)
-
+#define IRDMA_CQPSQ_MVPBP_INV_PD_ENT BIT_ULL(62)
#define IRDMA_CQPSQ_MVPBP_PD_PLPBA_S 3
-#define IRDMA_CQPSQ_MVPBP_PD_PLPBA_M \
- (0x1fffffffffffffffULL << IRDMA_CQPSQ_MVPBP_PD_PLPBA_S)
+#define IRDMA_CQPSQ_MVPBP_PD_PLPBA GENMASK_ULL(63, 3)
/* Manage Push Page - MPP */
#define IRDMA_INVALID_PUSH_PAGE_INDEX_GEN_1 0xffff
#define IRDMA_INVALID_PUSH_PAGE_INDEX 0xffffffff
#define IRDMA_CQPSQ_MPP_QS_HANDLE_S 0
-#define IRDMA_CQPSQ_MPP_QS_HANDLE_M \
- (0x3ffULL << IRDMA_CQPSQ_MPP_QS_HANDLE_S)
-
+#define IRDMA_CQPSQ_MPP_QS_HANDLE GENMASK_ULL(9, 0)
#define IRDMA_CQPSQ_MPP_PPIDX_S 0
-#define IRDMA_CQPSQ_MPP_PPIDX_M (0x3ffULL << IRDMA_CQPSQ_MPP_PPIDX_S)
-
+#define IRDMA_CQPSQ_MPP_PPIDX GENMASK_ULL(9, 0)
#define IRDMA_CQPSQ_MPP_PPTYPE_S 60
-#define IRDMA_CQPSQ_MPP_PPTYPE_M (0x3ULL << IRDMA_CQPSQ_MPP_PPTYPE_S)
+#define IRDMA_CQPSQ_MPP_PPTYPE GENMASK_ULL(61, 60)
#define IRDMA_CQPSQ_MPP_FREE_PAGE_S 62
-#define IRDMA_CQPSQ_MPP_FREE_PAGE_M BIT_ULL(IRDMA_CQPSQ_MPP_FREE_PAGE_S)
+#define IRDMA_CQPSQ_MPP_FREE_PAGE BIT_ULL(62)
/* Upload Context - UCTX */
#define IRDMA_CQPSQ_UCTX_QPCTXADDR_S IRDMA_CQPHC_QPCTX_S
-#define IRDMA_CQPSQ_UCTX_QPCTXADDR_M IRDMA_CQPHC_QPCTX_M
-
+#define IRDMA_CQPSQ_UCTX_QPCTXADDR IRDMA_CQPHC_QPCTX
#define IRDMA_CQPSQ_UCTX_QPID_S 0
-#define IRDMA_CQPSQ_UCTX_QPID_M (0xffffffULL << IRDMA_CQPSQ_UCTX_QPID_S)
-
+#define IRDMA_CQPSQ_UCTX_QPID GENMASK_ULL(23, 0)
#define IRDMA_CQPSQ_UCTX_QPTYPE_S 48
-#define IRDMA_CQPSQ_UCTX_QPTYPE_M (0xfULL << IRDMA_CQPSQ_UCTX_QPTYPE_S)
+#define IRDMA_CQPSQ_UCTX_QPTYPE GENMASK_ULL(51, 48)
#define IRDMA_CQPSQ_UCTX_RAWFORMAT_S 61
-#define IRDMA_CQPSQ_UCTX_RAWFORMAT_M \
- BIT_ULL(IRDMA_CQPSQ_UCTX_RAWFORMAT_S)
-
+#define IRDMA_CQPSQ_UCTX_RAWFORMAT BIT_ULL(61)
#define IRDMA_CQPSQ_UCTX_FREEZEQP_S 62
-#define IRDMA_CQPSQ_UCTX_FREEZEQP_M \
- BIT_ULL(IRDMA_CQPSQ_UCTX_FREEZEQP_S)
+#define IRDMA_CQPSQ_UCTX_FREEZEQP BIT_ULL(62)
-/* Manage HMC PM Function Table - MHMC */
#define IRDMA_CQPSQ_MHMC_VFIDX_S 0
-#define IRDMA_CQPSQ_MHMC_VFIDX_M (0xffffULL << IRDMA_CQPSQ_MHMC_VFIDX_S)
-
+#define IRDMA_CQPSQ_MHMC_VFIDX GENMASK_ULL(15, 0)
#define IRDMA_CQPSQ_MHMC_FREEPMFN_S 62
-#define IRDMA_CQPSQ_MHMC_FREEPMFN_M \
- BIT_ULL(IRDMA_CQPSQ_MHMC_FREEPMFN_S)
+#define IRDMA_CQPSQ_MHMC_FREEPMFN BIT_ULL(62)
-/* Set HMC Resource Profile - SHMCRP */
#define IRDMA_CQPSQ_SHMCRP_HMC_PROFILE_S 0
-#define IRDMA_CQPSQ_SHMCRP_HMC_PROFILE_M \
- (0x7ULL << IRDMA_CQPSQ_SHMCRP_HMC_PROFILE_S)
+#define IRDMA_CQPSQ_SHMCRP_HMC_PROFILE GENMASK_ULL(2, 0)
#define IRDMA_CQPSQ_SHMCRP_VFNUM_S 32
-#define IRDMA_CQPSQ_SHMCRP_VFNUM_M (0x3fULL << IRDMA_CQPSQ_SHMCRP_VFNUM_S)
-
-/* Create/Destroy CEQ */
+#define IRDMA_CQPSQ_SHMCRP_VFNUM GENMASK_ULL(37, 32)
#define IRDMA_CQPSQ_CEQ_CEQSIZE_S 0
-#define IRDMA_CQPSQ_CEQ_CEQSIZE_M \
- (0x3fffffULL << IRDMA_CQPSQ_CEQ_CEQSIZE_S)
-
+#define IRDMA_CQPSQ_CEQ_CEQSIZE GENMASK_ULL(21, 0)
#define IRDMA_CQPSQ_CEQ_CEQID_S 0
-#define IRDMA_CQPSQ_CEQ_CEQID_M (0x3ffULL << IRDMA_CQPSQ_CEQ_CEQID_S)
+#define IRDMA_CQPSQ_CEQ_CEQID GENMASK_ULL(9, 0)
#define IRDMA_CQPSQ_CEQ_LPBLSIZE_S IRDMA_CQPSQ_CQ_LPBLSIZE_S
#define IRDMA_CQPSQ_CEQ_LPBLSIZE_M IRDMA_CQPSQ_CQ_LPBLSIZE_M
-
+#define IRDMA_CQPSQ_CEQ_LPBLSIZE IRDMA_CQPSQ_CQ_LPBLSIZE
#define IRDMA_CQPSQ_CEQ_VMAP_S 47
-#define IRDMA_CQPSQ_CEQ_VMAP_M BIT_ULL(IRDMA_CQPSQ_CEQ_VMAP_S)
-
+#define IRDMA_CQPSQ_CEQ_VMAP BIT_ULL(47)
#define IRDMA_CQPSQ_CEQ_ITRNOEXPIRE_S 46
-#define IRDMA_CQPSQ_CEQ_ITRNOEXPIRE_M BIT_ULL(IRDMA_CQPSQ_CEQ_ITRNOEXPIRE_S)
-
+#define IRDMA_CQPSQ_CEQ_ITRNOEXPIRE BIT_ULL(46)
#define IRDMA_CQPSQ_CEQ_FIRSTPMPBLIDX_S 0
-#define IRDMA_CQPSQ_CEQ_FIRSTPMPBLIDX_M \
- (0xfffffffULL << IRDMA_CQPSQ_CEQ_FIRSTPMPBLIDX_S)
-
-/* Create/Destroy AEQ */
+#define IRDMA_CQPSQ_CEQ_FIRSTPMPBLIDX GENMASK_ULL(27, 0)
#define IRDMA_CQPSQ_AEQ_AEQECNT_S 0
-#define IRDMA_CQPSQ_AEQ_AEQECNT_M \
- (0x7ffffULL << IRDMA_CQPSQ_AEQ_AEQECNT_S)
+#define IRDMA_CQPSQ_AEQ_AEQECNT GENMASK_ULL(18, 0)
#define IRDMA_CQPSQ_AEQ_LPBLSIZE_S IRDMA_CQPSQ_CQ_LPBLSIZE_S
#define IRDMA_CQPSQ_AEQ_LPBLSIZE_M IRDMA_CQPSQ_CQ_LPBLSIZE_M
-
+#define IRDMA_CQPSQ_AEQ_LPBLSIZE IRDMA_CQPSQ_CQ_LPBLSIZE
#define IRDMA_CQPSQ_AEQ_VMAP_S 47
-#define IRDMA_CQPSQ_AEQ_VMAP_M BIT_ULL(IRDMA_CQPSQ_AEQ_VMAP_S)
-
+#define IRDMA_CQPSQ_AEQ_VMAP BIT_ULL(47)
#define IRDMA_CQPSQ_AEQ_FIRSTPMPBLIDX_S 0
-#define IRDMA_CQPSQ_AEQ_FIRSTPMPBLIDX_M \
- (0xfffffffULL << IRDMA_CQPSQ_AEQ_FIRSTPMPBLIDX_S)
+#define IRDMA_CQPSQ_AEQ_FIRSTPMPBLIDX GENMASK_ULL(27, 0)
-/* Commit FPM Values - CFPM */
#define IRDMA_COMMIT_FPM_QPCNT_S 0
-#define IRDMA_COMMIT_FPM_QPCNT_M (0x7ffffULL << IRDMA_COMMIT_FPM_QPCNT_S)
+#define IRDMA_COMMIT_FPM_QPCNT GENMASK_ULL(18, 0)
#define IRDMA_COMMIT_FPM_BASE_S 32
-
#define IRDMA_CQPSQ_CFPM_HMCFNID_S 0
-#define IRDMA_CQPSQ_CFPM_HMCFNID_M (0x3fULL << IRDMA_CQPSQ_CFPM_HMCFNID_S)
+#define IRDMA_CQPSQ_CFPM_HMCFNID GENMASK_ULL(5, 0)
-/* Flush WQEs - FWQE */
#define IRDMA_CQPSQ_FWQE_AECODE_S 0
-#define IRDMA_CQPSQ_FWQE_AECODE_M (0xffffULL << IRDMA_CQPSQ_FWQE_AECODE_S)
-
+#define IRDMA_CQPSQ_FWQE_AECODE GENMASK_ULL(15, 0)
#define IRDMA_CQPSQ_FWQE_AESOURCE_S 16
-#define IRDMA_CQPSQ_FWQE_AESOURCE_M \
- (0xfULL << IRDMA_CQPSQ_FWQE_AESOURCE_S)
-
+#define IRDMA_CQPSQ_FWQE_AESOURCE GENMASK_ULL(19, 16)
#define IRDMA_CQPSQ_FWQE_RQMNERR_S 0
-#define IRDMA_CQPSQ_FWQE_RQMNERR_M \
- (0xffffULL << IRDMA_CQPSQ_FWQE_RQMNERR_S)
-
+#define IRDMA_CQPSQ_FWQE_RQMNERR GENMASK_ULL(15, 0)
#define IRDMA_CQPSQ_FWQE_RQMJERR_S 16
-#define IRDMA_CQPSQ_FWQE_RQMJERR_M \
- (0xffffULL << IRDMA_CQPSQ_FWQE_RQMJERR_S)
-
+#define IRDMA_CQPSQ_FWQE_RQMJERR GENMASK_ULL(31, 16)
#define IRDMA_CQPSQ_FWQE_SQMNERR_S 32
-#define IRDMA_CQPSQ_FWQE_SQMNERR_M \
- (0xffffULL << IRDMA_CQPSQ_FWQE_SQMNERR_S)
-
+#define IRDMA_CQPSQ_FWQE_SQMNERR GENMASK_ULL(47, 32)
#define IRDMA_CQPSQ_FWQE_SQMJERR_S 48
-#define IRDMA_CQPSQ_FWQE_SQMJERR_M \
- (0xffffULL << IRDMA_CQPSQ_FWQE_SQMJERR_S)
-
+#define IRDMA_CQPSQ_FWQE_SQMJERR GENMASK_ULL(63, 48)
#define IRDMA_CQPSQ_FWQE_QPID_S 0
-#define IRDMA_CQPSQ_FWQE_QPID_M (0xffffffULL << IRDMA_CQPSQ_FWQE_QPID_S)
-
+#define IRDMA_CQPSQ_FWQE_QPID GENMASK_ULL(23, 0)
#define IRDMA_CQPSQ_FWQE_GENERATE_AE_S 59
-#define IRDMA_CQPSQ_FWQE_GENERATE_AE_M \
- BIT_ULL(IRDMA_CQPSQ_FWQE_GENERATE_AE_S)
-
+#define IRDMA_CQPSQ_FWQE_GENERATE_AE BIT_ULL(59)
#define IRDMA_CQPSQ_FWQE_USERFLCODE_S 60
-#define IRDMA_CQPSQ_FWQE_USERFLCODE_M \
- BIT_ULL(IRDMA_CQPSQ_FWQE_USERFLCODE_S)
-
+#define IRDMA_CQPSQ_FWQE_USERFLCODE BIT_ULL(60)
#define IRDMA_CQPSQ_FWQE_FLUSHSQ_S 61
-#define IRDMA_CQPSQ_FWQE_FLUSHSQ_M BIT_ULL(IRDMA_CQPSQ_FWQE_FLUSHSQ_S)
-
+#define IRDMA_CQPSQ_FWQE_FLUSHSQ BIT_ULL(61)
#define IRDMA_CQPSQ_FWQE_FLUSHRQ_S 62
-#define IRDMA_CQPSQ_FWQE_FLUSHRQ_M BIT_ULL(IRDMA_CQPSQ_FWQE_FLUSHRQ_S)
-
-/* Manage Accelerated Port Table - MAPT */
+#define IRDMA_CQPSQ_FWQE_FLUSHRQ BIT_ULL(62)
#define IRDMA_CQPSQ_MAPT_PORT_S 0
-#define IRDMA_CQPSQ_MAPT_PORT_M (0xffffULL << IRDMA_CQPSQ_MAPT_PORT_S)
-
+#define IRDMA_CQPSQ_MAPT_PORT GENMASK_ULL(15, 0)
#define IRDMA_CQPSQ_MAPT_ADDPORT_S 62
-#define IRDMA_CQPSQ_MAPT_ADDPORT_M BIT_ULL(IRDMA_CQPSQ_MAPT_ADDPORT_S)
-
-/* Update Protocol Engine SDs */
+#define IRDMA_CQPSQ_MAPT_ADDPORT BIT_ULL(62)
#define IRDMA_CQPSQ_UPESD_SDCMD_S 0
-#define IRDMA_CQPSQ_UPESD_SDCMD_M (0xffffffffULL << IRDMA_CQPSQ_UPESD_SDCMD_S)
-
+#define IRDMA_CQPSQ_UPESD_SDCMD GENMASK_ULL(31, 0)
#define IRDMA_CQPSQ_UPESD_SDDATALOW_S 0
-#define IRDMA_CQPSQ_UPESD_SDDATALOW_M \
- (0xffffffffULL << IRDMA_CQPSQ_UPESD_SDDATALOW_S)
-
+#define IRDMA_CQPSQ_UPESD_SDDATALOW GENMASK_ULL(31, 0)
#define IRDMA_CQPSQ_UPESD_SDDATAHI_S 32
-#define IRDMA_CQPSQ_UPESD_SDDATAHI_M \
- (0xffffffffULL << IRDMA_CQPSQ_UPESD_SDDATAHI_S)
-#define IRDMA_CQPSQ_UPESD_HMCFNID_S 0
-#define IRDMA_CQPSQ_UPESD_HMCFNID_M \
- (0x3fULL << IRDMA_CQPSQ_UPESD_HMCFNID_S)
-
+#define IRDMA_CQPSQ_UPESD_SDDATAHI GENMASK_ULL(63, 32)
#define IRDMA_CQPSQ_UPESD_ENTRY_VALID_S 63
-#define IRDMA_CQPSQ_UPESD_ENTRY_VALID_M \
- BIT_ULL(IRDMA_CQPSQ_UPESD_ENTRY_VALID_S)
+#define IRDMA_CQPSQ_UPESD_ENTRY_VALID BIT_ULL(63)
#define IRDMA_CQPSQ_UPESD_BM_PF 0
#define IRDMA_CQPSQ_UPESD_BM_CP_LM 1
#define IRDMA_CQPSQ_UPESD_BM_AXF 2
#define IRDMA_CQPSQ_UPESD_BM_LM 4
-
#define IRDMA_CQPSQ_UPESD_BM_S 32
-#define IRDMA_CQPSQ_UPESD_BM_M \
- (0x7ULL << IRDMA_CQPSQ_UPESD_BM_S)
-
+#define IRDMA_CQPSQ_UPESD_BM GENMASK_ULL(34, 32)
#define IRDMA_CQPSQ_UPESD_ENTRY_COUNT_S 0
-#define IRDMA_CQPSQ_UPESD_ENTRY_COUNT_M \
- (0xfULL << IRDMA_CQPSQ_UPESD_ENTRY_COUNT_S)
-
+#define IRDMA_CQPSQ_UPESD_ENTRY_COUNT GENMASK_ULL(3, 0)
#define IRDMA_CQPSQ_UPESD_SKIP_ENTRY_S 7
-#define IRDMA_CQPSQ_UPESD_SKIP_ENTRY_M \
- BIT_ULL(IRDMA_CQPSQ_UPESD_SKIP_ENTRY_S)
+#define IRDMA_CQPSQ_UPESD_SKIP_ENTRY BIT_ULL(7)
/* Suspend QP */
#define IRDMA_CQPSQ_SUSPENDQP_QPID_S 0
-#define IRDMA_CQPSQ_SUSPENDQP_QPID_M (0xFFFFFFULL)
-
-/* Resume QP */
+#define IRDMA_CQPSQ_SUSPENDQP_QPID GENMASK_ULL(23, 0)
#define IRDMA_CQPSQ_RESUMEQP_QSHANDLE_S 0
-#define IRDMA_CQPSQ_RESUMEQP_QSHANDLE_M \
- (0xffffffffULL << IRDMA_CQPSQ_RESUMEQP_QSHANDLE_S)
+#define IRDMA_CQPSQ_RESUMEQP_QSHANDLE GENMASK_ULL(31, 0)
-#define IRDMA_CQPSQ_RESUMEQP_QPID_S 0
-#define IRDMA_CQPSQ_RESUMEQP_QPID_M (0xFFFFFFUL)
+#define IRDMA_CQPSQ_RESUMEQP_QPID_S IRDMA_CQPSQ_SUSPENDQP_QPID_S
+#define IRDMA_CQPSQ_RESUMEQP_QPID_M IRDMA_CQPSQ_SUSPENDQP_QPID_M
+#define IRDMA_CQPSQ_RESUMEQP_QPID IRDMA_CQPSQ_SUSPENDQP_QPID
#define IRDMA_CQPSQ_MIN_STAG_INVALID 0x0001
#define IRDMA_CQPSQ_MIN_SUSPEND_PND 0x0005
@@ -1467,636 +955,422 @@
#define IRDMA_CQPSQ_MAJ_OBJCACHE_ERROR 0xF000
#define IRDMA_CQPSQ_MAJ_CNTXTCACHE_ERROR 0xF001
#define IRDMA_CQPSQ_MAJ_ERROR 0xFFFF
-
-/* IW QP Context */
#define IRDMAQPC_DDP_VER_S 0
-#define IRDMAQPC_DDP_VER_M (3ULL << IRDMAQPC_DDP_VER_S)
-
+#define IRDMAQPC_DDP_VER GENMASK_ULL(1, 0)
#define IRDMAQPC_IBRDENABLE_S 2
-#define IRDMAQPC_IBRDENABLE_M BIT_ULL(IRDMAQPC_IBRDENABLE_S)
-
+#define IRDMAQPC_IBRDENABLE BIT_ULL(2)
#define IRDMAQPC_IPV4_S 3
-#define IRDMAQPC_IPV4_M BIT_ULL(IRDMAQPC_IPV4_S)
-
+#define IRDMAQPC_IPV4 BIT_ULL(3)
#define IRDMAQPC_NONAGLE_S 4
-#define IRDMAQPC_NONAGLE_M BIT_ULL(IRDMAQPC_NONAGLE_S)
-
+#define IRDMAQPC_NONAGLE BIT_ULL(4)
#define IRDMAQPC_INSERTVLANTAG_S 5
-#define IRDMAQPC_INSERTVLANTAG_M BIT_ULL(IRDMAQPC_INSERTVLANTAG_S)
-
+#define IRDMAQPC_INSERTVLANTAG BIT_ULL(5)
#define IRDMAQPC_ISQP1_S 6
-#define IRDMAQPC_ISQP1_M BIT_ULL(IRDMAQPC_ISQP1_S)
-
+#define IRDMAQPC_ISQP1 BIT_ULL(6)
#define IRDMAQPC_TIMESTAMP_S 7
-#define IRDMAQPC_TIMESTAMP_M BIT_ULL(IRDMAQPC_TIMESTAMP_S)
-
+#define IRDMAQPC_TIMESTAMP BIT_ULL(7)
#define IRDMAQPC_RQWQESIZE_S 8
-#define IRDMAQPC_RQWQESIZE_M (3ULL << IRDMAQPC_RQWQESIZE_S)
-
+#define IRDMAQPC_RQWQESIZE GENMASK_ULL(9, 8)
#define IRDMAQPC_INSERTL2TAG2_S 11
-#define IRDMAQPC_INSERTL2TAG2_M BIT_ULL(IRDMAQPC_INSERTL2TAG2_S)
-
+#define IRDMAQPC_INSERTL2TAG2 BIT_ULL(11)
#define IRDMAQPC_LIMIT_S 12
-#define IRDMAQPC_LIMIT_M (3ULL << IRDMAQPC_LIMIT_S)
+#define IRDMAQPC_LIMIT GENMASK_ULL(13, 12)
#define IRDMAQPC_ECN_EN_S 14
-#define IRDMAQPC_ECN_EN_M BIT_ULL(IRDMAQPC_ECN_EN_S)
-
+#define IRDMAQPC_ECN_EN BIT_ULL(14)
#define IRDMAQPC_DROPOOOSEG_S 15
-#define IRDMAQPC_DROPOOOSEG_M BIT_ULL(IRDMAQPC_DROPOOOSEG_S)
-
+#define IRDMAQPC_DROPOOOSEG BIT_ULL(15)
#define IRDMAQPC_DUPACK_THRESH_S 16
-#define IRDMAQPC_DUPACK_THRESH_M (7ULL << IRDMAQPC_DUPACK_THRESH_S)
-
+#define IRDMAQPC_DUPACK_THRESH GENMASK_ULL(18, 16)
#define IRDMAQPC_ERR_RQ_IDX_VALID_S 19
-#define IRDMAQPC_ERR_RQ_IDX_VALID_M BIT_ULL(IRDMAQPC_ERR_RQ_IDX_VALID_S)
-
+#define IRDMAQPC_ERR_RQ_IDX_VALID BIT_ULL(19)
#define IRDMAQPC_DIS_VLAN_CHECKS_S 19
-#define IRDMAQPC_DIS_VLAN_CHECKS_M (7ULL << IRDMAQPC_DIS_VLAN_CHECKS_S)
-
+#define IRDMAQPC_DIS_VLAN_CHECKS GENMASK_ULL(21, 19)
#define IRDMAQPC_DC_TCP_EN_S 25
-#define IRDMAQPC_DC_TCP_EN_M BIT_ULL(IRDMAQPC_DC_TCP_EN_S)
-
+#define IRDMAQPC_DC_TCP_EN BIT_ULL(25)
#define IRDMAQPC_RCVTPHEN_S 28
-#define IRDMAQPC_RCVTPHEN_M BIT_ULL(IRDMAQPC_RCVTPHEN_S)
-
+#define IRDMAQPC_RCVTPHEN BIT_ULL(28)
#define IRDMAQPC_XMITTPHEN_S 29
-#define IRDMAQPC_XMITTPHEN_M BIT_ULL(IRDMAQPC_XMITTPHEN_S)
-
+#define IRDMAQPC_XMITTPHEN BIT_ULL(29)
#define IRDMAQPC_RQTPHEN_S 30
-#define IRDMAQPC_RQTPHEN_M BIT_ULL(IRDMAQPC_RQTPHEN_S)
-
+#define IRDMAQPC_RQTPHEN BIT_ULL(30)
#define IRDMAQPC_SQTPHEN_S 31
-#define IRDMAQPC_SQTPHEN_M BIT_ULL(IRDMAQPC_SQTPHEN_S)
-
+#define IRDMAQPC_SQTPHEN BIT_ULL(31)
#define IRDMAQPC_PPIDX_S 32
-#define IRDMAQPC_PPIDX_M (0x3ffULL << IRDMAQPC_PPIDX_S)
-
+#define IRDMAQPC_PPIDX GENMASK_ULL(41, 32)
#define IRDMAQPC_PMENA_S 47
-#define IRDMAQPC_PMENA_M BIT_ULL(IRDMAQPC_PMENA_S)
-
+#define IRDMAQPC_PMENA BIT_ULL(47)
#define IRDMAQPC_RDMAP_VER_S 62
-#define IRDMAQPC_RDMAP_VER_M (3ULL << IRDMAQPC_RDMAP_VER_S)
-
+#define IRDMAQPC_RDMAP_VER GENMASK_ULL(63, 62)
#define IRDMAQPC_ROCE_TVER_S 60
-#define IRDMAQPC_ROCE_TVER_M (0x0fULL << IRDMAQPC_ROCE_TVER_S)
+#define IRDMAQPC_ROCE_TVER GENMASK_ULL(63, 60)
#define IRDMAQPC_SQADDR_S IRDMA_CQPHC_QPCTX_S
-#define IRDMAQPC_SQADDR_M IRDMA_CQPHC_QPCTX_M
+#define IRDMAQPC_SQADDR IRDMA_CQPHC_QPCTX
#define IRDMAQPC_RQADDR_S IRDMA_CQPHC_QPCTX_S
-#define IRDMAQPC_RQADDR_M IRDMA_CQPHC_QPCTX_M
-
+#define IRDMAQPC_RQADDR IRDMA_CQPHC_QPCTX
#define IRDMAQPC_TTL_S 0
-#define IRDMAQPC_TTL_M (0xffULL << IRDMAQPC_TTL_S)
-
+#define IRDMAQPC_TTL GENMASK_ULL(7, 0)
#define IRDMAQPC_RQSIZE_S 8
-#define IRDMAQPC_RQSIZE_M (0xfULL << IRDMAQPC_RQSIZE_S)
-
+#define IRDMAQPC_RQSIZE GENMASK_ULL(11, 8)
#define IRDMAQPC_SQSIZE_S 12
-#define IRDMAQPC_SQSIZE_M (0xfULL << IRDMAQPC_SQSIZE_S)
-
+#define IRDMAQPC_SQSIZE GENMASK_ULL(15, 12)
#define IRDMAQPC_GEN1_SRCMACADDRIDX_S 16
-#define IRDMAQPC_GEN1_SRCMACADDRIDX_M (0x3fUL << IRDMAQPC_GEN1_SRCMACADDRIDX_S)
-
+#define IRDMAQPC_GEN1_SRCMACADDRIDX GENMASK(21, 16)
#define IRDMAQPC_AVOIDSTRETCHACK_S 23
-#define IRDMAQPC_AVOIDSTRETCHACK_M BIT_ULL(IRDMAQPC_AVOIDSTRETCHACK_S)
-
+#define IRDMAQPC_AVOIDSTRETCHACK BIT_ULL(23)
#define IRDMAQPC_TOS_S 24
-#define IRDMAQPC_TOS_M (0xffULL << IRDMAQPC_TOS_S)
-
+#define IRDMAQPC_TOS GENMASK_ULL(31, 24)
#define IRDMAQPC_SRCPORTNUM_S 32
-#define IRDMAQPC_SRCPORTNUM_M (0xffffULL << IRDMAQPC_SRCPORTNUM_S)
-
+#define IRDMAQPC_SRCPORTNUM GENMASK_ULL(47, 32)
#define IRDMAQPC_DESTPORTNUM_S 48
-#define IRDMAQPC_DESTPORTNUM_M (0xffffULL << IRDMAQPC_DESTPORTNUM_S)
-
+#define IRDMAQPC_DESTPORTNUM GENMASK_ULL(63, 48)
#define IRDMAQPC_DESTIPADDR0_S 32
-#define IRDMAQPC_DESTIPADDR0_M \
- (0xffffffffULL << IRDMAQPC_DESTIPADDR0_S)
-
+#define IRDMAQPC_DESTIPADDR0 GENMASK_ULL(63, 32)
#define IRDMAQPC_DESTIPADDR1_S 0
-#define IRDMAQPC_DESTIPADDR1_M \
- (0xffffffffULL << IRDMAQPC_DESTIPADDR1_S)
-
+#define IRDMAQPC_DESTIPADDR1 GENMASK_ULL(31, 0)
#define IRDMAQPC_DESTIPADDR2_S 32
-#define IRDMAQPC_DESTIPADDR2_M \
- (0xffffffffULL << IRDMAQPC_DESTIPADDR2_S)
-
+#define IRDMAQPC_DESTIPADDR2 GENMASK_ULL(63, 32)
#define IRDMAQPC_DESTIPADDR3_S 0
-#define IRDMAQPC_DESTIPADDR3_M \
- (0xffffffffULL << IRDMAQPC_DESTIPADDR3_S)
-
+#define IRDMAQPC_DESTIPADDR3 GENMASK_ULL(31, 0)
#define IRDMAQPC_SNDMSS_S 16
-#define IRDMAQPC_SNDMSS_M (0x3fffULL << IRDMAQPC_SNDMSS_S)
-
+#define IRDMAQPC_SNDMSS GENMASK_ULL(29, 16)
#define IRDMAQPC_SYN_RST_HANDLING_S 30
-#define IRDMAQPC_SYN_RST_HANDLING_M (0x3ULL << IRDMAQPC_SYN_RST_HANDLING_S)
-
+#define IRDMAQPC_SYN_RST_HANDLING GENMASK_ULL(31, 30)
#define IRDMAQPC_VLANTAG_S 32
-#define IRDMAQPC_VLANTAG_M (0xffffULL << IRDMAQPC_VLANTAG_S)
-
+#define IRDMAQPC_VLANTAG GENMASK_ULL(47, 32)
#define IRDMAQPC_ARPIDX_S 48
-#define IRDMAQPC_ARPIDX_M (0xffffULL << IRDMAQPC_ARPIDX_S)
-
+#define IRDMAQPC_ARPIDX GENMASK_ULL(63, 48)
#define IRDMAQPC_FLOWLABEL_S 0
-#define IRDMAQPC_FLOWLABEL_M (0xfffffULL << IRDMAQPC_FLOWLABEL_S)
-
+#define IRDMAQPC_FLOWLABEL GENMASK_ULL(19, 0)
#define IRDMAQPC_WSCALE_S 20
-#define IRDMAQPC_WSCALE_M BIT_ULL(IRDMAQPC_WSCALE_S)
-
+#define IRDMAQPC_WSCALE BIT_ULL(20)
#define IRDMAQPC_KEEPALIVE_S 21
-#define IRDMAQPC_KEEPALIVE_M BIT_ULL(IRDMAQPC_KEEPALIVE_S)
-
+#define IRDMAQPC_KEEPALIVE BIT_ULL(21)
#define IRDMAQPC_IGNORE_TCP_OPT_S 22
-#define IRDMAQPC_IGNORE_TCP_OPT_M BIT_ULL(IRDMAQPC_IGNORE_TCP_OPT_S)
-
+#define IRDMAQPC_IGNORE_TCP_OPT BIT_ULL(22)
#define IRDMAQPC_IGNORE_TCP_UNS_OPT_S 23
-#define IRDMAQPC_IGNORE_TCP_UNS_OPT_M \
- BIT_ULL(IRDMAQPC_IGNORE_TCP_UNS_OPT_S)
-
+#define IRDMAQPC_IGNORE_TCP_UNS_OPT BIT_ULL(23)
#define IRDMAQPC_TCPSTATE_S 28
-#define IRDMAQPC_TCPSTATE_M (0xfULL << IRDMAQPC_TCPSTATE_S)
-
+#define IRDMAQPC_TCPSTATE GENMASK_ULL(31, 28)
#define IRDMAQPC_RCVSCALE_S 32
-#define IRDMAQPC_RCVSCALE_M (0xfULL << IRDMAQPC_RCVSCALE_S)
-
+#define IRDMAQPC_RCVSCALE GENMASK_ULL(35, 32)
#define IRDMAQPC_SNDSCALE_S 40
-#define IRDMAQPC_SNDSCALE_M (0xfULL << IRDMAQPC_SNDSCALE_S)
-
+#define IRDMAQPC_SNDSCALE GENMASK_ULL(43, 40)
#define IRDMAQPC_PDIDX_S 48
-#define IRDMAQPC_PDIDX_M (0xffffULL << IRDMAQPC_PDIDX_S)
-
+#define IRDMAQPC_PDIDX GENMASK_ULL(63, 48)
#define IRDMAQPC_PDIDXHI_S 20
-#define IRDMAQPC_PDIDXHI_M (0x3ULL << IRDMAQPC_PDIDXHI_S)
-
+#define IRDMAQPC_PDIDXHI GENMASK_ULL(21, 20)
#define IRDMAQPC_PKEY_S 32
-#define IRDMAQPC_PKEY_M (0xffffULL << IRDMAQPC_PKEY_S)
-
+#define IRDMAQPC_PKEY GENMASK_ULL(47, 32)
#define IRDMAQPC_ACKCREDITS_S 20
-#define IRDMAQPC_ACKCREDITS_M (0x1fULL << IRDMAQPC_ACKCREDITS_S)
-
+#define IRDMAQPC_ACKCREDITS GENMASK_ULL(24, 20)
#define IRDMAQPC_QKEY_S 32
-#define IRDMAQPC_QKEY_M (0xffffffffULL << IRDMAQPC_QKEY_S)
-
+#define IRDMAQPC_QKEY GENMASK_ULL(63, 32)
#define IRDMAQPC_DESTQP_S 0
-#define IRDMAQPC_DESTQP_M (0xffffffULL << IRDMAQPC_DESTQP_S)
-
+#define IRDMAQPC_DESTQP GENMASK_ULL(23, 0)
#define IRDMAQPC_KALIVE_TIMER_MAX_PROBES_S 16
-#define IRDMAQPC_KALIVE_TIMER_MAX_PROBES_M \
- (0xffULL << IRDMAQPC_KALIVE_TIMER_MAX_PROBES_S)
-
+#define IRDMAQPC_KALIVE_TIMER_MAX_PROBES GENMASK_ULL(23, 16)
#define IRDMAQPC_KEEPALIVE_INTERVAL_S 24
-#define IRDMAQPC_KEEPALIVE_INTERVAL_M \
- (0xffULL << IRDMAQPC_KEEPALIVE_INTERVAL_S)
-
+#define IRDMAQPC_KEEPALIVE_INTERVAL GENMASK_ULL(31, 24)
#define IRDMAQPC_TIMESTAMP_RECENT_S 0
-#define IRDMAQPC_TIMESTAMP_RECENT_M \
- (0xffffffffULL << IRDMAQPC_TIMESTAMP_RECENT_S)
-
+#define IRDMAQPC_TIMESTAMP_RECENT GENMASK_ULL(31, 0)
#define IRDMAQPC_TIMESTAMP_AGE_S 32
-#define IRDMAQPC_TIMESTAMP_AGE_M \
- (0xffffffffULL << IRDMAQPC_TIMESTAMP_AGE_S)
-
+#define IRDMAQPC_TIMESTAMP_AGE GENMASK_ULL(63, 32)
#define IRDMAQPC_SNDNXT_S 0
-#define IRDMAQPC_SNDNXT_M (0xffffffffULL << IRDMAQPC_SNDNXT_S)
-
+#define IRDMAQPC_SNDNXT GENMASK_ULL(31, 0)
#define IRDMAQPC_ISN_S 32
-#define IRDMAQPC_ISN_M (0x00ffffffULL << IRDMAQPC_ISN_S)
-
+#define IRDMAQPC_ISN GENMASK_ULL(55, 32)
#define IRDMAQPC_PSNNXT_S 0
-#define IRDMAQPC_PSNNXT_M (0x00ffffffULL << IRDMAQPC_PSNNXT_S)
-
+#define IRDMAQPC_PSNNXT GENMASK_ULL(23, 0)
#define IRDMAQPC_LSN_S 32
-#define IRDMAQPC_LSN_M (0x00ffffffULL << IRDMAQPC_LSN_S)
-
+#define IRDMAQPC_LSN GENMASK_ULL(55, 32)
#define IRDMAQPC_SNDWND_S 32
-#define IRDMAQPC_SNDWND_M (0xffffffffULL << IRDMAQPC_SNDWND_S)
-
+#define IRDMAQPC_SNDWND GENMASK_ULL(63, 32)
#define IRDMAQPC_RCVNXT_S 0
-#define IRDMAQPC_RCVNXT_M (0xffffffffULL << IRDMAQPC_RCVNXT_S)
-
+#define IRDMAQPC_RCVNXT GENMASK_ULL(31, 0)
#define IRDMAQPC_EPSN_S 0
-#define IRDMAQPC_EPSN_M (0x00ffffffULL << IRDMAQPC_EPSN_S)
-
+#define IRDMAQPC_EPSN GENMASK_ULL(23, 0)
#define IRDMAQPC_RCVWND_S 32
-#define IRDMAQPC_RCVWND_M (0xffffffffULL << IRDMAQPC_RCVWND_S)
-
+#define IRDMAQPC_RCVWND GENMASK_ULL(63, 32)
#define IRDMAQPC_SNDMAX_S 0
-#define IRDMAQPC_SNDMAX_M (0xffffffffULL << IRDMAQPC_SNDMAX_S)
-
+#define IRDMAQPC_SNDMAX GENMASK_ULL(31, 0)
#define IRDMAQPC_SNDUNA_S 32
-#define IRDMAQPC_SNDUNA_M (0xffffffffULL << IRDMAQPC_SNDUNA_S)
-
+#define IRDMAQPC_SNDUNA GENMASK_ULL(63, 32)
#define IRDMAQPC_PSNMAX_S 0
-#define IRDMAQPC_PSNMAX_M (0x00ffffffULL << IRDMAQPC_PSNMAX_S)
+#define IRDMAQPC_PSNMAX GENMASK_ULL(23, 0)
#define IRDMAQPC_PSNUNA_S 32
-#define IRDMAQPC_PSNUNA_M (0xffffffULL << IRDMAQPC_PSNUNA_S)
-
+#define IRDMAQPC_PSNUNA GENMASK_ULL(55, 32)
#define IRDMAQPC_SRTT_S 0
-#define IRDMAQPC_SRTT_M (0xffffffffULL << IRDMAQPC_SRTT_S)
-
+#define IRDMAQPC_SRTT GENMASK_ULL(31, 0)
#define IRDMAQPC_RTTVAR_S 32
-#define IRDMAQPC_RTTVAR_M (0xffffffffULL << IRDMAQPC_RTTVAR_S)
-
+#define IRDMAQPC_RTTVAR GENMASK_ULL(63, 32)
#define IRDMAQPC_SSTHRESH_S 0
-#define IRDMAQPC_SSTHRESH_M (0xffffffffULL << IRDMAQPC_SSTHRESH_S)
-
+#define IRDMAQPC_SSTHRESH GENMASK_ULL(31, 0)
#define IRDMAQPC_CWND_S 32
-#define IRDMAQPC_CWND_M (0xffffffffULL << IRDMAQPC_CWND_S)
-
+#define IRDMAQPC_CWND GENMASK_ULL(63, 32)
#define IRDMAQPC_CWNDROCE_S 32
-#define IRDMAQPC_CWNDROCE_M (0xffffffULL << IRDMAQPC_CWNDROCE_S)
-
+#define IRDMAQPC_CWNDROCE GENMASK_ULL(55, 32)
#define IRDMAQPC_SNDWL1_S 0
-#define IRDMAQPC_SNDWL1_M (0xffffffffULL << IRDMAQPC_SNDWL1_S)
-
+#define IRDMAQPC_SNDWL1 GENMASK_ULL(31, 0)
#define IRDMAQPC_SNDWL2_S 32
-#define IRDMAQPC_SNDWL2_M (0xffffffffULL << IRDMAQPC_SNDWL2_S)
-
+#define IRDMAQPC_SNDWL2 GENMASK_ULL(63, 32)
#define IRDMAQPC_ERR_RQ_IDX_S 32
-#define IRDMAQPC_ERR_RQ_IDX_M (0x3fffULL << IRDMAQPC_ERR_RQ_IDX_S)
-
+#define IRDMAQPC_ERR_RQ_IDX GENMASK_ULL(46, 32)
#define IRDMAQPC_RTOMIN_S 57
-#define IRDMAQPC_RTOMIN_M (0x7fULL << IRDMAQPC_RTOMIN_S)
-
+#define IRDMAQPC_RTOMIN GENMASK_ULL(63, 57)
#define IRDMAQPC_MAXSNDWND_S 0
-#define IRDMAQPC_MAXSNDWND_M (0xffffffffULL << IRDMAQPC_MAXSNDWND_S)
-
+#define IRDMAQPC_MAXSNDWND GENMASK_ULL(31, 0)
#define IRDMAQPC_REXMIT_THRESH_S 48
-#define IRDMAQPC_REXMIT_THRESH_M (0x3fULL << IRDMAQPC_REXMIT_THRESH_S)
-
+#define IRDMAQPC_REXMIT_THRESH GENMASK_ULL(53, 48)
#define IRDMAQPC_RNRNAK_THRESH_S 54
-#define IRDMAQPC_RNRNAK_THRESH_M (0x7ULL << IRDMAQPC_RNRNAK_THRESH_S)
-
+#define IRDMAQPC_RNRNAK_THRESH GENMASK_ULL(56, 54)
#define IRDMAQPC_TXCQNUM_S 0
-#define IRDMAQPC_TXCQNUM_M (0x7ffffULL << IRDMAQPC_TXCQNUM_S)
-
+#define IRDMAQPC_TXCQNUM GENMASK_ULL(18, 0)
#define IRDMAQPC_RXCQNUM_S 32
-#define IRDMAQPC_RXCQNUM_M (0x7ffffULL << IRDMAQPC_RXCQNUM_S)
-
+#define IRDMAQPC_RXCQNUM GENMASK_ULL(50, 32)
#define IRDMAQPC_STAT_INDEX_S 0
-#define IRDMAQPC_STAT_INDEX_M (0x7fULL << IRDMAQPC_STAT_INDEX_S)
-
+#define IRDMAQPC_STAT_INDEX GENMASK_ULL(6, 0)
#define IRDMAQPC_Q2ADDR_S 8
-#define IRDMAQPC_Q2ADDR_M (0xffffffffffffffULL << IRDMAQPC_Q2ADDR_S)
-
+#define IRDMAQPC_Q2ADDR GENMASK_ULL(63, 8)
#define IRDMAQPC_LASTBYTESENT_S 0
-#define IRDMAQPC_LASTBYTESENT_M (0xffULL << IRDMAQPC_LASTBYTESENT_S)
-
+#define IRDMAQPC_LASTBYTESENT GENMASK_ULL(7, 0)
#define IRDMAQPC_MACADDRESS_S 16
-#define IRDMAQPC_MACADDRESS_M (0xffffffffffffULL << IRDMAQPC_MACADDRESS_S)
-
+#define IRDMAQPC_MACADDRESS GENMASK_ULL(63, 16)
#define IRDMAQPC_ORDSIZE_S 0
-#define IRDMAQPC_ORDSIZE_M (0xffULL << IRDMAQPC_ORDSIZE_S)
+#define IRDMAQPC_ORDSIZE GENMASK_ULL(7, 0)
#define IRDMAQPC_IRDSIZE_S 16
-#define IRDMAQPC_IRDSIZE_M (0x7ULL << IRDMAQPC_IRDSIZE_S)
+#define IRDMAQPC_IRDSIZE GENMASK_ULL(18, 16)
#define IRDMAQPC_UDPRIVCQENABLE_S 19
-#define IRDMAQPC_UDPRIVCQENABLE_M BIT_ULL(IRDMAQPC_UDPRIVCQENABLE_S)
-
+#define IRDMAQPC_UDPRIVCQENABLE BIT_ULL(19)
#define IRDMAQPC_WRRDRSPOK_S 20
-#define IRDMAQPC_WRRDRSPOK_M BIT_ULL(IRDMAQPC_WRRDRSPOK_S)
-
+#define IRDMAQPC_WRRDRSPOK BIT_ULL(20)
#define IRDMAQPC_RDOK_S 21
-#define IRDMAQPC_RDOK_M BIT_ULL(IRDMAQPC_RDOK_S)
-
+#define IRDMAQPC_RDOK BIT_ULL(21)
#define IRDMAQPC_SNDMARKERS_S 22
-#define IRDMAQPC_SNDMARKERS_M BIT_ULL(IRDMAQPC_SNDMARKERS_S)
-
+#define IRDMAQPC_SNDMARKERS BIT_ULL(22)
#define IRDMAQPC_DCQCNENABLE_S 22
-#define IRDMAQPC_DCQCNENABLE_M BIT_ULL(IRDMAQPC_DCQCNENABLE_S)
-
+#define IRDMAQPC_DCQCNENABLE BIT_ULL(22)
#define IRDMAQPC_FW_CC_ENABLE_S 28
-#define IRDMAQPC_FW_CC_ENABLE_M BIT_ULL(IRDMAQPC_FW_CC_ENABLE_S)
-
+#define IRDMAQPC_FW_CC_ENABLE BIT_ULL(28)
#define IRDMAQPC_RCVNOICRC_S 31
-#define IRDMAQPC_RCVNOICRC_M BIT_ULL(IRDMAQPC_RCVNOICRC_S)
-
+#define IRDMAQPC_RCVNOICRC BIT_ULL(31)
#define IRDMAQPC_BINDEN_S 23
-#define IRDMAQPC_BINDEN_M BIT_ULL(IRDMAQPC_BINDEN_S)
-
+#define IRDMAQPC_BINDEN BIT_ULL(23)
#define IRDMAQPC_FASTREGEN_S 24
-#define IRDMAQPC_FASTREGEN_M BIT_ULL(IRDMAQPC_FASTREGEN_S)
-
+#define IRDMAQPC_FASTREGEN BIT_ULL(24)
#define IRDMAQPC_PRIVEN_S 25
-#define IRDMAQPC_PRIVEN_M BIT_ULL(IRDMAQPC_PRIVEN_S)
-
+#define IRDMAQPC_PRIVEN BIT_ULL(25)
#define IRDMAQPC_TIMELYENABLE_S 27
-#define IRDMAQPC_TIMELYENABLE_M BIT_ULL(IRDMAQPC_TIMELYENABLE_S)
-
+#define IRDMAQPC_TIMELYENABLE BIT_ULL(27)
#define IRDMAQPC_THIGH_S 52
-#define IRDMAQPC_THIGH_M ((u64)0xfff << IRDMAQPC_THIGH_S)
-
+#define IRDMAQPC_THIGH GENMASK_ULL(63, 52)
#define IRDMAQPC_TLOW_S 32
-#define IRDMAQPC_TLOW_M ((u64)0xFF << IRDMAQPC_TLOW_S)
-
+#define IRDMAQPC_TLOW GENMASK_ULL(39, 32)
#define IRDMAQPC_REMENDPOINTIDX_S 0
-#define IRDMAQPC_REMENDPOINTIDX_M ((u64)0x1FFFF << IRDMAQPC_REMENDPOINTIDX_S)
-
+#define IRDMAQPC_REMENDPOINTIDX GENMASK_ULL(16, 0)
#define IRDMAQPC_USESTATSINSTANCE_S 26
-#define IRDMAQPC_USESTATSINSTANCE_M BIT_ULL(IRDMAQPC_USESTATSINSTANCE_S)
-
+#define IRDMAQPC_USESTATSINSTANCE BIT_ULL(26)
#define IRDMAQPC_IWARPMODE_S 28
-#define IRDMAQPC_IWARPMODE_M BIT_ULL(IRDMAQPC_IWARPMODE_S)
-
+#define IRDMAQPC_IWARPMODE BIT_ULL(28)
#define IRDMAQPC_RCVMARKERS_S 29
-#define IRDMAQPC_RCVMARKERS_M BIT_ULL(IRDMAQPC_RCVMARKERS_S)
-
+#define IRDMAQPC_RCVMARKERS BIT_ULL(29)
#define IRDMAQPC_ALIGNHDRS_S 30
-#define IRDMAQPC_ALIGNHDRS_M BIT_ULL(IRDMAQPC_ALIGNHDRS_S)
-
+#define IRDMAQPC_ALIGNHDRS BIT_ULL(30)
#define IRDMAQPC_RCVNOMPACRC_S 31
-#define IRDMAQPC_RCVNOMPACRC_M BIT_ULL(IRDMAQPC_RCVNOMPACRC_S)
-
+#define IRDMAQPC_RCVNOMPACRC BIT_ULL(31)
#define IRDMAQPC_RCVMARKOFFSET_S 32
-#define IRDMAQPC_RCVMARKOFFSET_M (0x1ffULL << IRDMAQPC_RCVMARKOFFSET_S)
-
+#define IRDMAQPC_RCVMARKOFFSET GENMASK_ULL(40, 32)
#define IRDMAQPC_SNDMARKOFFSET_S 48
-#define IRDMAQPC_SNDMARKOFFSET_M (0x1ffULL << IRDMAQPC_SNDMARKOFFSET_S)
+#define IRDMAQPC_SNDMARKOFFSET GENMASK_ULL(56, 48)
#define IRDMAQPC_QPCOMPCTX_S IRDMA_CQPHC_QPCTX_S
-#define IRDMAQPC_QPCOMPCTX_M IRDMA_CQPHC_QPCTX_M
-
+#define IRDMAQPC_QPCOMPCTX IRDMA_CQPHC_QPCTX
#define IRDMAQPC_SQTPHVAL_S 0
-#define IRDMAQPC_SQTPHVAL_M (0xffULL << IRDMAQPC_SQTPHVAL_S)
-
+#define IRDMAQPC_SQTPHVAL GENMASK_ULL(7, 0)
#define IRDMAQPC_RQTPHVAL_S 8
-#define IRDMAQPC_RQTPHVAL_M (0xffULL << IRDMAQPC_RQTPHVAL_S)
-
+#define IRDMAQPC_RQTPHVAL GENMASK_ULL(15, 8)
#define IRDMAQPC_QSHANDLE_S 16
-#define IRDMAQPC_QSHANDLE_M (0x3ffULL << IRDMAQPC_QSHANDLE_S)
-
+#define IRDMAQPC_QSHANDLE GENMASK_ULL(25, 16)
#define IRDMAQPC_EXCEPTION_LAN_QUEUE_S 32
-#define IRDMAQPC_EXCEPTION_LAN_QUEUE_M \
- (0xfffULL << IRDMAQPC_EXCEPTION_LAN_QUEUE_S)
-
+#define IRDMAQPC_EXCEPTION_LAN_QUEUE GENMASK_ULL(43, 32)
#define IRDMAQPC_LOCAL_IPADDR3_S 0
-#define IRDMAQPC_LOCAL_IPADDR3_M \
- (0xffffffffULL << IRDMAQPC_LOCAL_IPADDR3_S)
-
+#define IRDMAQPC_LOCAL_IPADDR3 GENMASK_ULL(31, 0)
#define IRDMAQPC_LOCAL_IPADDR2_S 32
-#define IRDMAQPC_LOCAL_IPADDR2_M \
- (0xffffffffULL << IRDMAQPC_LOCAL_IPADDR2_S)
-
+#define IRDMAQPC_LOCAL_IPADDR2 GENMASK_ULL(63, 32)
#define IRDMAQPC_LOCAL_IPADDR1_S 0
-#define IRDMAQPC_LOCAL_IPADDR1_M \
- (0xffffffffULL << IRDMAQPC_LOCAL_IPADDR1_S)
-
+#define IRDMAQPC_LOCAL_IPADDR1 GENMASK_ULL(31, 0)
#define IRDMAQPC_LOCAL_IPADDR0_S 32
-#define IRDMAQPC_LOCAL_IPADDR0_M \
- (0xffffffffULL << IRDMAQPC_LOCAL_IPADDR0_S)
-
+#define IRDMAQPC_LOCAL_IPADDR0 GENMASK_ULL(63, 32)
#define IRDMA_FW_VER_MINOR_S 0
-#define IRDMA_FW_VER_MINOR_M \
- (0xffffULL << IRDMA_FW_VER_MINOR_S)
-
+#define IRDMA_FW_VER_MINOR GENMASK_ULL(15, 0)
#define IRDMA_FW_VER_MAJOR_S 16
-#define IRDMA_FW_VER_MAJOR_M \
- (0xffffULL << IRDMA_FW_VER_MAJOR_S)
-
+#define IRDMA_FW_VER_MAJOR GENMASK_ULL(31, 16)
#define IRDMA_FEATURE_INFO_S 0
-#define IRDMA_FEATURE_INFO_M \
- (0xffffffffffffULL << IRDMA_FEATURE_INFO_S)
-
+#define IRDMA_FEATURE_INFO GENMASK_ULL(47, 0)
#define IRDMA_FEATURE_CNT_S 32
-#define IRDMA_FEATURE_CNT_M \
- (0xffffULL << IRDMA_FEATURE_CNT_S)
-
+#define IRDMA_FEATURE_CNT GENMASK_ULL(47, 32)
#define IRDMA_FEATURE_TYPE_S 48
-#define IRDMA_FEATURE_TYPE_M \
- (0xffffULL << IRDMA_FEATURE_TYPE_S)
-
+#define IRDMA_FEATURE_TYPE GENMASK_ULL(63, 48)
#define IRDMA_RSVD_S 41
-#define IRDMA_RSVD_M (0x7fffULL << IRDMA_RSVD_S)
+#define IRDMA_RSVD GENMASK_ULL(55, 41)
-/* iwarp QP SQ WQE common fields */
#define IRDMAQPSQ_OPCODE_S 32
-#define IRDMAQPSQ_OPCODE_M (0x3fULL << IRDMAQPSQ_OPCODE_S)
-
+#define IRDMAQPSQ_OPCODE GENMASK_ULL(37, 32)
#define IRDMAQPSQ_COPY_HOST_PBL_S 43
-#define IRDMAQPSQ_COPY_HOST_PBL_M BIT_ULL(IRDMAQPSQ_COPY_HOST_PBL_S)
-
+#define IRDMAQPSQ_COPY_HOST_PBL BIT_ULL(43)
#define IRDMAQPSQ_ADDFRAGCNT_S 38
-#define IRDMAQPSQ_ADDFRAGCNT_M (0xfULL << IRDMAQPSQ_ADDFRAGCNT_S)
-
+#define IRDMAQPSQ_ADDFRAGCNT GENMASK_ULL(41, 38)
#define IRDMAQPSQ_PUSHWQE_S 56
-#define IRDMAQPSQ_PUSHWQE_M BIT_ULL(IRDMAQPSQ_PUSHWQE_S)
-
+#define IRDMAQPSQ_PUSHWQE BIT_ULL(56)
#define IRDMAQPSQ_STREAMMODE_S 58
-#define IRDMAQPSQ_STREAMMODE_M BIT_ULL(IRDMAQPSQ_STREAMMODE_S)
-
+#define IRDMAQPSQ_STREAMMODE BIT_ULL(58)
#define IRDMAQPSQ_WAITFORRCVPDU_S 59
-#define IRDMAQPSQ_WAITFORRCVPDU_M BIT_ULL(IRDMAQPSQ_WAITFORRCVPDU_S)
-
+#define IRDMAQPSQ_WAITFORRCVPDU BIT_ULL(59)
#define IRDMAQPSQ_READFENCE_S 60
-#define IRDMAQPSQ_READFENCE_M BIT_ULL(IRDMAQPSQ_READFENCE_S)
-
+#define IRDMAQPSQ_READFENCE BIT_ULL(60)
#define IRDMAQPSQ_LOCALFENCE_S 61
-#define IRDMAQPSQ_LOCALFENCE_M BIT_ULL(IRDMAQPSQ_LOCALFENCE_S)
-
+#define IRDMAQPSQ_LOCALFENCE BIT_ULL(61)
#define IRDMAQPSQ_UDPHEADER_S 61
-#define IRDMAQPSQ_UDPHEADER_M BIT_ULL(IRDMAQPSQ_UDPHEADER_S)
-
+#define IRDMAQPSQ_UDPHEADER BIT_ULL(61)
#define IRDMAQPSQ_L4LEN_S 42
-#define IRDMAQPSQ_L4LEN_M ((u64)0xF << IRDMAQPSQ_L4LEN_S)
-
+#define IRDMAQPSQ_L4LEN GENMASK_ULL(45, 42)
#define IRDMAQPSQ_SIGCOMPL_S 62
-#define IRDMAQPSQ_SIGCOMPL_M BIT_ULL(IRDMAQPSQ_SIGCOMPL_S)
-
+#define IRDMAQPSQ_SIGCOMPL BIT_ULL(62)
#define IRDMAQPSQ_VALID_S 63
-#define IRDMAQPSQ_VALID_M BIT_ULL(IRDMAQPSQ_VALID_S)
+#define IRDMAQPSQ_VALID BIT_ULL(63)
#define IRDMAQPSQ_FRAG_TO_S IRDMA_CQPHC_QPCTX_S
-#define IRDMAQPSQ_FRAG_TO_M IRDMA_CQPHC_QPCTX_M
-
+#define IRDMAQPSQ_FRAG_TO IRDMA_CQPHC_QPCTX
#define IRDMAQPSQ_FRAG_VALID_S 63
-#define IRDMAQPSQ_FRAG_VALID_M BIT_ULL(IRDMAQPSQ_FRAG_VALID_S)
-
+#define IRDMAQPSQ_FRAG_VALID BIT_ULL(63)
#define IRDMAQPSQ_FRAG_LEN_S 32
-#define IRDMAQPSQ_FRAG_LEN_M (0x7fffffffULL << IRDMAQPSQ_FRAG_LEN_S)
-
+#define IRDMAQPSQ_FRAG_LEN GENMASK_ULL(62, 32)
#define IRDMAQPSQ_FRAG_STAG_S 0
-#define IRDMAQPSQ_FRAG_STAG_M (0xffffffffULL << IRDMAQPSQ_FRAG_STAG_S)
-
+#define IRDMAQPSQ_FRAG_STAG GENMASK_ULL(31, 0)
#define IRDMAQPSQ_GEN1_FRAG_LEN_S 0
-#define IRDMAQPSQ_GEN1_FRAG_LEN_M (0xffffffffULL << IRDMAQPSQ_GEN1_FRAG_LEN_S)
-
+#define IRDMAQPSQ_GEN1_FRAG_LEN GENMASK_ULL(31, 0)
#define IRDMAQPSQ_GEN1_FRAG_STAG_S 32
-#define IRDMAQPSQ_GEN1_FRAG_STAG_M (0xffffffffULL << IRDMAQPSQ_GEN1_FRAG_STAG_S)
-
+#define IRDMAQPSQ_GEN1_FRAG_STAG GENMASK_ULL(63, 32)
#define IRDMAQPSQ_REMSTAGINV_S 0
-#define IRDMAQPSQ_REMSTAGINV_M (0xffffffffULL << IRDMAQPSQ_REMSTAGINV_S)
-
+#define IRDMAQPSQ_REMSTAGINV GENMASK_ULL(31, 0)
#define IRDMAQPSQ_DESTQKEY_S 0
-#define IRDMAQPSQ_DESTQKEY_M (0xffffffffULL << IRDMAQPSQ_DESTQKEY_S)
-
+#define IRDMAQPSQ_DESTQKEY GENMASK_ULL(31, 0)
#define IRDMAQPSQ_DESTQPN_S 32
-#define IRDMAQPSQ_DESTQPN_M (0x00ffffffULL << IRDMAQPSQ_DESTQPN_S)
-
+#define IRDMAQPSQ_DESTQPN GENMASK_ULL(55, 32)
#define IRDMAQPSQ_AHID_S 0
-#define IRDMAQPSQ_AHID_M (0x0001ffffULL << IRDMAQPSQ_AHID_S)
-
+#define IRDMAQPSQ_AHID GENMASK_ULL(16, 0)
#define IRDMAQPSQ_INLINEDATAFLAG_S 57
-#define IRDMAQPSQ_INLINEDATAFLAG_M BIT_ULL(IRDMAQPSQ_INLINEDATAFLAG_S)
+#define IRDMAQPSQ_INLINEDATAFLAG BIT_ULL(57)
#define IRDMA_INLINE_VALID_S 7
-
#define IRDMAQPSQ_INLINEDATALEN_S 48
-#define IRDMAQPSQ_INLINEDATALEN_M \
- (0xffULL << IRDMAQPSQ_INLINEDATALEN_S)
+#define IRDMAQPSQ_INLINEDATALEN GENMASK_ULL(55, 48)
#define IRDMAQPSQ_IMMDATAFLAG_S 47
-#define IRDMAQPSQ_IMMDATAFLAG_M \
- BIT_ULL(IRDMAQPSQ_IMMDATAFLAG_S)
+#define IRDMAQPSQ_IMMDATAFLAG BIT_ULL(47)
#define IRDMAQPSQ_REPORTRTT_S 46
-#define IRDMAQPSQ_REPORTRTT_M \
- BIT_ULL(IRDMAQPSQ_REPORTRTT_S)
+#define IRDMAQPSQ_REPORTRTT BIT_ULL(46)
#define IRDMAQPSQ_IMMDATA_S 0
-#define IRDMAQPSQ_IMMDATA_M \
- (0xffffffffffffffffULL << IRDMAQPSQ_IMMDATA_S)
-
-/* rdma write */
+#define IRDMAQPSQ_IMMDATA GENMASK_ULL(63, 0)
#define IRDMAQPSQ_REMSTAG_S 0
-#define IRDMAQPSQ_REMSTAG_M (0xffffffffULL << IRDMAQPSQ_REMSTAG_S)
+#define IRDMAQPSQ_REMSTAG GENMASK_ULL(31, 0)
#define IRDMAQPSQ_REMTO_S IRDMA_CQPHC_QPCTX_S
-#define IRDMAQPSQ_REMTO_M IRDMA_CQPHC_QPCTX_M
+#define IRDMAQPSQ_REMTO IRDMA_CQPHC_QPCTX
-/* memory window */
#define IRDMAQPSQ_STAGRIGHTS_S 48
-#define IRDMAQPSQ_STAGRIGHTS_M (0x1fULL << IRDMAQPSQ_STAGRIGHTS_S)
-
+#define IRDMAQPSQ_STAGRIGHTS GENMASK_ULL(52, 48)
#define IRDMAQPSQ_VABASEDTO_S 53
-#define IRDMAQPSQ_VABASEDTO_M BIT_ULL(IRDMAQPSQ_VABASEDTO_S)
-
+#define IRDMAQPSQ_VABASEDTO BIT_ULL(53)
#define IRDMAQPSQ_MEMWINDOWTYPE_S 54
-#define IRDMAQPSQ_MEMWINDOWTYPE_M BIT_ULL(IRDMAQPSQ_MEMWINDOWTYPE_S)
+#define IRDMAQPSQ_MEMWINDOWTYPE BIT_ULL(54)
#define IRDMAQPSQ_MWLEN_S IRDMA_CQPHC_QPCTX_S
-#define IRDMAQPSQ_MWLEN_M IRDMA_CQPHC_QPCTX_M
-
+#define IRDMAQPSQ_MWLEN IRDMA_CQPHC_QPCTX
#define IRDMAQPSQ_PARENTMRSTAG_S 32
-#define IRDMAQPSQ_PARENTMRSTAG_M \
- (0xffffffffULL << IRDMAQPSQ_PARENTMRSTAG_S)
-
+#define IRDMAQPSQ_PARENTMRSTAG GENMASK_ULL(63, 32)
#define IRDMAQPSQ_MWSTAG_S 0
-#define IRDMAQPSQ_MWSTAG_M (0xffffffffULL << IRDMAQPSQ_MWSTAG_S)
+#define IRDMAQPSQ_MWSTAG GENMASK_ULL(31, 0)
#define IRDMAQPSQ_BASEVA_TO_FBO_S IRDMA_CQPHC_QPCTX_S
-#define IRDMAQPSQ_BASEVA_TO_FBO_M IRDMA_CQPHC_QPCTX_M
+#define IRDMAQPSQ_BASEVA_TO_FBO IRDMA_CQPHC_QPCTX
-/* Local Invalidate */
#define IRDMAQPSQ_LOCSTAG_S 0
-#define IRDMAQPSQ_LOCSTAG_M (0xffffffffULL << IRDMAQPSQ_LOCSTAG_S)
+#define IRDMAQPSQ_LOCSTAG GENMASK_ULL(31, 0)
-/* Fast Register */
#define IRDMAQPSQ_STAGKEY_S 0
-#define IRDMAQPSQ_STAGKEY_M (0xffULL << IRDMAQPSQ_STAGKEY_S)
-
+#define IRDMAQPSQ_STAGKEY GENMASK_ULL(7, 0)
#define IRDMAQPSQ_STAGINDEX_S 8
-#define IRDMAQPSQ_STAGINDEX_M (0xffffffULL << IRDMAQPSQ_STAGINDEX_S)
-
+#define IRDMAQPSQ_STAGINDEX GENMASK_ULL(31, 8)
#define IRDMAQPSQ_COPYHOSTPBLS_S 43
-#define IRDMAQPSQ_COPYHOSTPBLS_M BIT_ULL(IRDMAQPSQ_COPYHOSTPBLS_S)
-
+#define IRDMAQPSQ_COPYHOSTPBLS BIT_ULL(43)
#define IRDMAQPSQ_LPBLSIZE_S 44
-#define IRDMAQPSQ_LPBLSIZE_M (3ULL << IRDMAQPSQ_LPBLSIZE_S)
-
+#define IRDMAQPSQ_LPBLSIZE GENMASK_ULL(45, 44)
#define IRDMAQPSQ_HPAGESIZE_S 46
-#define IRDMAQPSQ_HPAGESIZE_M (3ULL << IRDMAQPSQ_HPAGESIZE_S)
-
+#define IRDMAQPSQ_HPAGESIZE GENMASK_ULL(47, 46)
#define IRDMAQPSQ_STAGLEN_S 0
-#define IRDMAQPSQ_STAGLEN_M (0x1ffffffffffULL << IRDMAQPSQ_STAGLEN_S)
-
+#define IRDMAQPSQ_STAGLEN GENMASK_ULL(40, 0)
#define IRDMAQPSQ_FIRSTPMPBLIDXLO_S 48
-#define IRDMAQPSQ_FIRSTPMPBLIDXLO_M \
- (0xffffULL << IRDMAQPSQ_FIRSTPMPBLIDXLO_S)
-
+#define IRDMAQPSQ_FIRSTPMPBLIDXLO GENMASK_ULL(63, 48)
#define IRDMAQPSQ_FIRSTPMPBLIDXHI_S 0
-#define IRDMAQPSQ_FIRSTPMPBLIDXHI_M \
- (0xfffULL << IRDMAQPSQ_FIRSTPMPBLIDXHI_S)
-
+#define IRDMAQPSQ_FIRSTPMPBLIDXHI GENMASK_ULL(11, 0)
#define IRDMAQPSQ_PBLADDR_S 12
-#define IRDMAQPSQ_PBLADDR_M (0xfffffffffffffULL << IRDMAQPSQ_PBLADDR_S)
+#define IRDMAQPSQ_PBLADDR GENMASK_ULL(63, 12)
/* iwarp QP RQ WQE common fields */
#define IRDMAQPRQ_ADDFRAGCNT_S IRDMAQPSQ_ADDFRAGCNT_S
-#define IRDMAQPRQ_ADDFRAGCNT_M IRDMAQPSQ_ADDFRAGCNT_M
+#define IRDMAQPRQ_ADDFRAGCNT IRDMAQPSQ_ADDFRAGCNT
#define IRDMAQPRQ_VALID_S IRDMAQPSQ_VALID_S
-#define IRDMAQPRQ_VALID_M IRDMAQPSQ_VALID_M
+#define IRDMAQPRQ_VALID IRDMAQPSQ_VALID
#define IRDMAQPRQ_COMPLCTX_S IRDMA_CQPHC_QPCTX_S
-#define IRDMAQPRQ_COMPLCTX_M IRDMA_CQPHC_QPCTX_M
+#define IRDMAQPRQ_COMPLCTX IRDMA_CQPHC_QPCTX
#define IRDMAQPRQ_FRAG_LEN_S IRDMAQPSQ_FRAG_LEN_S
-#define IRDMAQPRQ_FRAG_LEN_M IRDMAQPSQ_FRAG_LEN_M
+#define IRDMAQPRQ_FRAG_LEN IRDMAQPSQ_FRAG_LEN
#define IRDMAQPRQ_STAG_S IRDMAQPSQ_FRAG_STAG_S
-#define IRDMAQPRQ_STAG_M IRDMAQPSQ_FRAG_STAG_M
+#define IRDMAQPRQ_STAG IRDMAQPSQ_FRAG_STAG
#define IRDMAQPRQ_TO_S IRDMAQPSQ_FRAG_TO_S
-#define IRDMAQPRQ_TO_M IRDMAQPSQ_FRAG_TO_M
+#define IRDMAQPRQ_TO IRDMAQPSQ_FRAG_TO
#define IRDMAPFINT_OICR_HMC_ERR_M BIT(26)
#define IRDMAPFINT_OICR_PE_PUSH_M BIT(27)
#define IRDMAPFINT_OICR_PE_CRITERR_M BIT(28)
-/* Query FPM CQP buf */
#define IRDMA_QUERY_FPM_MAX_QPS_S 0
-#define IRDMA_QUERY_FPM_MAX_QPS_M \
- (0x7ffffULL << IRDMA_QUERY_FPM_MAX_QPS_S)
-
+#define IRDMA_QUERY_FPM_MAX_QPS GENMASK_ULL(18, 0)
#define IRDMA_QUERY_FPM_MAX_CQS_S 0
-#define IRDMA_QUERY_FPM_MAX_CQS_M \
- (0xfffffULL << IRDMA_QUERY_FPM_MAX_CQS_S)
-
+#define IRDMA_QUERY_FPM_MAX_CQS GENMASK_ULL(19, 0)
#define IRDMA_QUERY_FPM_FIRST_PE_SD_INDEX_S 0
-#define IRDMA_QUERY_FPM_FIRST_PE_SD_INDEX_M \
- (0x3fffULL << IRDMA_QUERY_FPM_FIRST_PE_SD_INDEX_S)
-
+#define IRDMA_QUERY_FPM_FIRST_PE_SD_INDEX GENMASK_ULL(13, 0)
#define IRDMA_QUERY_FPM_MAX_PE_SDS_S 32
-#define IRDMA_QUERY_FPM_MAX_PE_SDS_M \
- (0x3fffULL << IRDMA_QUERY_FPM_MAX_PE_SDS_S)
+#define IRDMA_QUERY_FPM_MAX_PE_SDS GENMASK_ULL(45, 32)
#define IRDMA_QUERY_FPM_MAX_CEQS_S 0
-#define IRDMA_QUERY_FPM_MAX_CEQS_M \
- (0x3ffULL << IRDMA_QUERY_FPM_MAX_CEQS_S)
-
+#define IRDMA_QUERY_FPM_MAX_CEQS GENMASK_ULL(9, 0)
#define IRDMA_QUERY_FPM_XFBLOCKSIZE_S 32
-#define IRDMA_QUERY_FPM_XFBLOCKSIZE_M \
- (0xffffffffULL << IRDMA_QUERY_FPM_XFBLOCKSIZE_S)
-
+#define IRDMA_QUERY_FPM_XFBLOCKSIZE GENMASK_ULL(63, 32)
#define IRDMA_QUERY_FPM_Q1BLOCKSIZE_S 32
-#define IRDMA_QUERY_FPM_Q1BLOCKSIZE_M \
- (0xffffffffULL << IRDMA_QUERY_FPM_Q1BLOCKSIZE_S)
-
+#define IRDMA_QUERY_FPM_Q1BLOCKSIZE GENMASK_ULL(63, 32)
#define IRDMA_QUERY_FPM_HTMULTIPLIER_S 16
-#define IRDMA_QUERY_FPM_HTMULTIPLIER_M \
- (0xfULL << IRDMA_QUERY_FPM_HTMULTIPLIER_S)
-
+#define IRDMA_QUERY_FPM_HTMULTIPLIER GENMASK_ULL(19, 16)
#define IRDMA_QUERY_FPM_TIMERBUCKET_S 32
-#define IRDMA_QUERY_FPM_TIMERBUCKET_M \
- (0xffFFULL << IRDMA_QUERY_FPM_TIMERBUCKET_S)
-
+#define IRDMA_QUERY_FPM_TIMERBUCKET GENMASK_ULL(47, 32)
#define IRDMA_QUERY_FPM_RRFBLOCKSIZE_S 32
-#define IRDMA_QUERY_FPM_RRFBLOCKSIZE_M \
- (0xffffffffULL << IRDMA_QUERY_FPM_RRFBLOCKSIZE_S)
-
+#define IRDMA_QUERY_FPM_RRFBLOCKSIZE GENMASK_ULL(63, 32)
#define IRDMA_QUERY_FPM_RRFFLBLOCKSIZE_S 32
-#define IRDMA_QUERY_FPM_RRFFLBLOCKSIZE_M \
- (0xffffffffULL << IRDMA_QUERY_FPM_RRFFLBLOCKSIZE_S)
-
+#define IRDMA_QUERY_FPM_RRFFLBLOCKSIZE GENMASK_ULL(63, 32)
#define IRDMA_QUERY_FPM_OOISCFBLOCKSIZE_S 32
-#define IRDMA_QUERY_FPM_OOISCFBLOCKSIZE_M \
- (0xffffffffULL << IRDMA_QUERY_FPM_OOISCFBLOCKSIZE_S)
-
-/* Static HMC pages allocated buf */
+#define IRDMA_QUERY_FPM_OOISCFBLOCKSIZE GENMASK_ULL(63, 32)
#define IRDMA_SHMC_PAGE_ALLOCATED_HMC_FN_ID_S 0
-#define IRDMA_SHMC_PAGE_ALLOCATED_HMC_FN_ID_M \
- (0x3fULL << IRDMA_SHMC_PAGE_ALLOCATED_HMC_FN_ID_S)
+#define IRDMA_SHMC_PAGE_ALLOCATED_HMC_FN_ID GENMASK_ULL(15, 0)
#define IRDMA_GET_CURRENT_AEQ_ELEM(_aeq) \
( \
@@ -2251,6 +1525,12 @@
IRDMA_RING_MOVE_HEAD(_ring, _retcode); \
}
+enum irdma_protocol_used {
+ IRDMA_ANY_PROTOCOL = 0,
+ IRDMA_IWARP_PROTOCOL_ONLY = 1,
+ IRDMA_ROCE_PROTOCOL_ONLY = 2,
+};
+
enum irdma_qp_wqe_size {
IRDMA_WQE_SIZE_32 = 32,
IRDMA_WQE_SIZE_64 = 64,
diff --git a/sys/dev/irdma/irdma_hmc.h b/sys/dev/irdma/irdma_hmc.h
--- a/sys/dev/irdma/irdma_hmc.h
+++ b/sys/dev/irdma/irdma_hmc.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2021 Intel Corporation
+ * Copyright (c) 2015 - 2022 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -128,7 +128,7 @@
struct irdma_hmc_info {
u32 signature;
- u8 hmc_fn_id;
+ u16 hmc_fn_id;
u16 first_sd_index;
struct irdma_hmc_obj_info *hmc_obj;
struct irdma_virt_mem hmc_obj_virt_mem;
@@ -143,7 +143,7 @@
struct irdma_update_sds_info {
u32 cnt;
- u8 hmc_fn_id;
+ u16 hmc_fn_id;
struct irdma_update_sd_entry entry[IRDMA_MAX_SD_ENTRIES];
};
@@ -180,15 +180,15 @@
struct irdma_hmc_create_obj_info *info);
int irdma_sc_del_hmc_obj(struct irdma_sc_dev *dev,
struct irdma_hmc_del_obj_info *info, bool reset);
-int irdma_hmc_sd_one(struct irdma_sc_dev *dev, u8 hmc_fn_id, u64 pa, u32 sd_idx,
+int irdma_hmc_sd_one(struct irdma_sc_dev *dev, u16 hmc_fn_id, u64 pa, u32 sd_idx,
enum irdma_sd_entry_type type,
bool setsd);
int irdma_update_sds_noccq(struct irdma_sc_dev *dev,
struct irdma_update_sds_info *info);
struct irdma_vfdev *irdma_vfdev_from_fpm(struct irdma_sc_dev *dev,
- u8 hmc_fn_id);
+ u16 hmc_fn_id);
struct irdma_hmc_info *irdma_vf_hmcinfo_from_fpm(struct irdma_sc_dev *dev,
- u8 hmc_fn_id);
+ u16 hmc_fn_id);
int irdma_add_sd_table_entry(struct irdma_hw *hw,
struct irdma_hmc_info *hmc_info, u32 sd_index,
enum irdma_sd_entry_type type, u64 direct_mode_sz);
diff --git a/sys/dev/irdma/irdma_hmc.c b/sys/dev/irdma/irdma_hmc.c
--- a/sys/dev/irdma/irdma_hmc.c
+++ b/sys/dev/irdma/irdma_hmc.c
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2021 Intel Corporation
+ * Copyright (c) 2015 - 2022 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -106,10 +106,14 @@
irdma_set_sd_entry(u64 pa, u32 idx, enum irdma_sd_entry_type type,
struct irdma_update_sd_entry *entry)
{
- entry->data = pa | (IRDMA_HMC_MAX_BP_COUNT << IRDMA_PFHMC_SDDATALOW_PMSDBPCOUNT_S) |
- (((type == IRDMA_SD_TYPE_PAGED) ? 0 : 1) << IRDMA_PFHMC_SDDATALOW_PMSDTYPE_S) |
- (1 << IRDMA_PFHMC_SDDATALOW_PMSDVALID_S);
- entry->cmd = (idx | (1 << IRDMA_PFHMC_SDCMD_PMSDWR_S) | (1 << 15));
+ entry->data = pa |
+ FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDBPCOUNT, IRDMA_HMC_MAX_BP_COUNT) |
+ FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDTYPE,
+ type == IRDMA_SD_TYPE_PAGED ? 0 : 1) |
+ FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDVALID, 1);
+
+ entry->cmd = idx | FIELD_PREP(IRDMA_PFHMC_SDCMD_PMSDWR, 1) |
+ IRDMA_PFHMC_SDCMD_PMSDPARTSEL;
}
/**
@@ -122,9 +126,12 @@
irdma_clr_sd_entry(u32 idx, enum irdma_sd_entry_type type,
struct irdma_update_sd_entry *entry)
{
- entry->data = (IRDMA_HMC_MAX_BP_COUNT << IRDMA_PFHMC_SDDATALOW_PMSDBPCOUNT_S) |
- (((type == IRDMA_SD_TYPE_PAGED) ? 0 : 1) << IRDMA_PFHMC_SDDATALOW_PMSDTYPE_S);
- entry->cmd = (idx | (1 << IRDMA_PFHMC_SDCMD_PMSDWR_S) | (1 << 15));
+ entry->data = FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDBPCOUNT, IRDMA_HMC_MAX_BP_COUNT) |
+ FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDTYPE,
+ type == IRDMA_SD_TYPE_PAGED ? 0 : 1);
+
+ entry->cmd = idx | FIELD_PREP(IRDMA_PFHMC_SDCMD_PMSDWR, 1) |
+ IRDMA_PFHMC_SDCMD_PMSDPARTSEL;
}
/**
@@ -137,9 +144,9 @@
irdma_invalidate_pf_hmc_pd(struct irdma_sc_dev *dev, u32 sd_idx,
u32 pd_idx)
{
- u32 val = LS_32(sd_idx, IRDMA_PFHMC_PDINV_PMSDIDX) |
- LS_32(1, IRDMA_PFHMC_PDINV_PMSDPARTSEL) |
- LS_32(pd_idx, IRDMA_PFHMC_PDINV_PMPDIDX);
+ u32 val = FIELD_PREP(IRDMA_PFHMC_PDINV_PMSDIDX, sd_idx) |
+ FIELD_PREP(IRDMA_PFHMC_PDINV_PMSDPARTSEL, 1) |
+ FIELD_PREP(IRDMA_PFHMC_PDINV_PMPDIDX, pd_idx);
writel(val, dev->hw_regs[IRDMA_PFHMC_PDINV]);
}
@@ -154,7 +161,7 @@
* @setsd: flag to set or clear sd
*/
int
-irdma_hmc_sd_one(struct irdma_sc_dev *dev, u8 hmc_fn_id, u64 pa, u32 sd_idx,
+irdma_hmc_sd_one(struct irdma_sc_dev *dev, u16 hmc_fn_id, u64 pa, u32 sd_idx,
enum irdma_sd_entry_type type, bool setsd)
{
struct irdma_update_sds_info sdinfo;
@@ -534,7 +541,7 @@
&sd_entry->u.pd_table.pd_entry_virt_mem;
vmem->size = sizeof(struct irdma_hmc_pd_entry) * 512;
- vmem->va = kzalloc(vmem->size, GFP_ATOMIC);
+ vmem->va = kzalloc(vmem->size, GFP_KERNEL);
if (!vmem->va) {
irdma_free_dma_mem(hw, &dma_mem);
return -ENOMEM;
diff --git a/sys/dev/irdma/irdma_hw.c b/sys/dev/irdma/irdma_hw.c
--- a/sys/dev/irdma/irdma_hw.c
+++ b/sys/dev/irdma/irdma_hw.c
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2021 Intel Corporation
+ * Copyright (c) 2015 - 2022 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -74,6 +74,7 @@
IRDMA_HMC_IW_XFFL,
IRDMA_HMC_IW_Q1,
IRDMA_HMC_IW_Q1FL,
+ IRDMA_HMC_IW_PBLE,
IRDMA_HMC_IW_TIMER,
IRDMA_HMC_IW_FSIMC,
IRDMA_HMC_IW_FSIAV,
@@ -95,7 +96,7 @@
struct irdma_cq *cq = iwcq->back_cq;
if (!cq->user_mode)
- cq->armed = false;
+ atomic_set(&cq->armed, 0);
if (cq->ibcq.comp_handler)
cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
}
@@ -171,68 +172,36 @@
irdma_set_flush_fields(struct irdma_sc_qp *qp,
struct irdma_aeqe_info *info)
{
+ struct qp_err_code qp_err;
+
qp->sq_flush_code = info->sq;
qp->rq_flush_code = info->rq;
- qp->event_type = IRDMA_QP_EVENT_CATASTROPHIC;
- switch (info->ae_id) {
- case IRDMA_AE_AMP_BOUNDS_VIOLATION:
- case IRDMA_AE_AMP_INVALID_STAG:
- qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR;
- /* fallthrough */
- case IRDMA_AE_UDA_XMIT_BAD_PD:
- qp->flush_code = FLUSH_PROT_ERR;
- break;
- case IRDMA_AE_AMP_UNALLOCATED_STAG:
- case IRDMA_AE_AMP_BAD_PD:
- qp->flush_code = FLUSH_PROT_ERR;
- break;
- case IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG:
- case IRDMA_AE_AMP_BAD_QP:
- case IRDMA_AE_WQE_UNEXPECTED_OPCODE:
- qp->flush_code = FLUSH_LOC_QP_OP_ERR;
- break;
- case IRDMA_AE_AMP_BAD_STAG_KEY:
- case IRDMA_AE_AMP_BAD_STAG_INDEX:
- case IRDMA_AE_AMP_TO_WRAP:
- case IRDMA_AE_AMP_RIGHTS_VIOLATION:
- case IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
- case IRDMA_AE_PRIV_OPERATION_DENIED:
- case IRDMA_AE_IB_REMOTE_ACCESS_ERROR:
- case IRDMA_AE_IB_REMOTE_OP_ERROR:
- qp->flush_code = FLUSH_REM_ACCESS_ERR;
- qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR;
- break;
- case IRDMA_AE_LLP_SEGMENT_TOO_SMALL:
- case IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
- case IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT:
- case IRDMA_AE_UDA_L4LEN_INVALID:
- case IRDMA_AE_ROCE_RSP_LENGTH_ERROR:
- qp->flush_code = FLUSH_LOC_LEN_ERR;
- break;
- case IRDMA_AE_LCE_QP_CATASTROPHIC:
- qp->flush_code = FLUSH_FATAL_ERR;
- break;
- case IRDMA_AE_DDP_UBE_INVALID_MO:
- case IRDMA_AE_IB_RREQ_AND_Q1_FULL:
- case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR:
- qp->flush_code = FLUSH_GENERAL_ERR;
- break;
- case IRDMA_AE_AMP_MWBIND_INVALID_RIGHTS:
- case IRDMA_AE_AMP_MWBIND_BIND_DISABLED:
- case IRDMA_AE_AMP_MWBIND_INVALID_BOUNDS:
- qp->flush_code = FLUSH_MW_BIND_ERR;
- break;
- case IRDMA_AE_LLP_TOO_MANY_RETRIES:
- qp->flush_code = FLUSH_RETRY_EXC_ERR;
- break;
- case IRDMA_AE_IB_INVALID_REQUEST:
- qp->flush_code = FLUSH_REM_INV_REQ_ERR;
- break;
- default:
- qp->flush_code = FLUSH_FATAL_ERR;
- break;
+ qp_err = irdma_ae_to_qp_err_code(info->ae_id);
+
+ qp->flush_code = qp_err.flush_code;
+ qp->event_type = qp_err.event_type;
+}
+
+/**
+ * irdma_complete_cqp_request - perform post-completion cleanup
+ * @cqp: device CQP
+ * @cqp_request: CQP request
+ *
+ * Mark CQP request as done, wake up waiting thread or invoke
+ * callback function and release/free CQP request.
+ */
+static void
+irdma_complete_cqp_request(struct irdma_cqp *cqp,
+ struct irdma_cqp_request *cqp_request)
+{
+ if (cqp_request->waiting) {
+ cqp_request->request_done = true;
+ wake_up(&cqp_request->waitq);
+ } else if (cqp_request->callback_fcn) {
+ cqp_request->callback_fcn(cqp_request);
}
+ irdma_put_cqp_request(cqp, cqp_request);
}
/**
@@ -315,13 +284,11 @@
wake_up_interruptible(&iwqp->waitq);
break;
case IRDMA_AE_LLP_FIN_RECEIVED:
- case IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE:
if (qp->term_flags)
break;
if (atomic_inc_return(&iwqp->close_timer_started) == 1) {
iwqp->hw_tcp_state = IRDMA_TCP_STATE_CLOSE_WAIT;
- if (iwqp->hw_tcp_state == IRDMA_TCP_STATE_CLOSE_WAIT &&
- iwqp->ibqp_state == IB_QPS_RTS) {
+ if (iwqp->ibqp_state == IB_QPS_RTS) {
irdma_next_iw_state(iwqp,
IRDMA_QP_STATE_CLOSING,
0, 0, 0);
@@ -394,6 +361,7 @@
case IRDMA_AE_RESOURCE_EXHAUSTION:
break;
case IRDMA_AE_PRIV_OPERATION_DENIED:
+ case IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE:
case IRDMA_AE_STAG_ZERO_INVALID:
case IRDMA_AE_IB_RREQ_AND_Q1_FULL:
case IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION:
@@ -442,10 +410,7 @@
irdma_next_iw_state(iwqp, IRDMA_QP_STATE_ERROR, 1, 0, 0);
irdma_cm_disconn(iwqp);
} else {
- iwqp->sc_qp.term_flags = 1;
- irdma_next_iw_state(iwqp, IRDMA_QP_STATE_ERROR, 1, 0,
- 0);
- irdma_cm_disconn(iwqp);
+ irdma_terminate_connection(qp, info);
}
break;
}
@@ -513,8 +478,10 @@
u32 i;
u32 size;
- if (!rf->msix_count)
+ if (!rf->msix_count) {
+ irdma_dev_err(&rf->sc_dev, "No MSI-X vectors reserved for RDMA.\n");
return -EINVAL;
+ }
size = sizeof(struct irdma_msix_vector) * rf->msix_count;
size += sizeof(struct irdma_qvlist_info);
@@ -546,7 +513,7 @@
iw_qvinfo->aeq_idx = IRDMA_Q_INVALID_IDX;
iw_qvinfo->ceq_idx = ceq_idx++;
}
- iw_qvinfo->itr_idx = 3;
+ iw_qvinfo->itr_idx = IRDMA_IDX_NOITR;
iw_qvinfo->v_idx = rf->iw_msixtbl[i].idx;
}
@@ -636,8 +603,7 @@
if (rf->cqp_cmpl_wq)
destroy_workqueue(rf->cqp_cmpl_wq);
- if (free_hwcqp)
- status = irdma_sc_cqp_destroy(dev->cqp);
+ status = irdma_sc_cqp_destroy(dev->cqp, free_hwcqp);
if (status)
irdma_debug(dev, IRDMA_DEBUG_ERR, "Destroy CQP failed %d\n",
status);
@@ -898,6 +864,8 @@
info.entry_type = rf->sd_type;
for (i = 0; i < IW_HMC_OBJ_TYPE_NUM; i++) {
+ if (iw_hmc_obj_types[i] == IRDMA_HMC_IW_PBLE)
+ continue;
if (dev->hmc_info->hmc_obj[iw_hmc_obj_types[i]].cnt) {
info.rsrc_type = iw_hmc_obj_types[i];
info.count = dev->hmc_info->hmc_obj[info.rsrc_type].cnt;
@@ -992,8 +960,8 @@
cqp->scratch_array = kcalloc(sqsize, sizeof(*cqp->scratch_array), GFP_KERNEL);
memset(cqp->scratch_array, 0, sqsize * sizeof(*cqp->scratch_array));
if (!cqp->scratch_array) {
- kfree(cqp->cqp_requests);
- return -ENOMEM;
+ status = -ENOMEM;
+ goto err_scratch;
}
dev->cqp = &cqp->sc_cqp;
@@ -1002,15 +970,14 @@
cqp->sq.va = irdma_allocate_dma_mem(dev->hw, &cqp->sq, cqp->sq.size,
IRDMA_CQP_ALIGNMENT);
if (!cqp->sq.va) {
- kfree(cqp->scratch_array);
- kfree(cqp->cqp_requests);
- return -ENOMEM;
+ status = -ENOMEM;
+ goto err_sq;
}
status = irdma_obj_aligned_mem(rf, &mem, sizeof(struct irdma_cqp_ctx),
IRDMA_HOST_CTX_ALIGNMENT_M);
if (status)
- goto exit;
+ goto err_ctx;
dev->cqp->host_ctx_pa = mem.pa;
dev->cqp->host_ctx = mem.va;
@@ -1040,7 +1007,7 @@
if (status) {
irdma_debug(dev, IRDMA_DEBUG_ERR, "cqp init status %d\n",
status);
- goto exit;
+ goto err_ctx;
}
spin_lock_init(&cqp->req_lock);
@@ -1051,7 +1018,7 @@
irdma_debug(dev, IRDMA_DEBUG_ERR,
"cqp create failed - status %d maj_err %d min_err %d\n",
status, maj_err, min_err);
- goto exit;
+ goto err_create;
}
INIT_LIST_HEAD(&cqp->cqp_avail_reqs);
@@ -1065,8 +1032,15 @@
init_waitqueue_head(&cqp->remove_wq);
return 0;
-exit:
- irdma_destroy_cqp(rf, false);
+err_create:
+err_ctx:
+ irdma_free_dma_mem(dev->hw, &cqp->sq);
+err_sq:
+ kfree(cqp->scratch_array);
+ cqp->scratch_array = NULL;
+err_scratch:
+ kfree(cqp->cqp_requests);
+ cqp->cqp_requests = NULL;
return status;
}
@@ -1224,12 +1198,6 @@
return status;
bus_describe_intr(rf->dev_ctx.dev, msix_vec->res, msix_vec->tag, "CEQ");
}
- status = bus_bind_intr(rf->dev_ctx.dev, msix_vec->res, msix_vec->cpu_affinity);
- if (status) {
- irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR,
- "ceq irq config fail\n");
- return status;
- }
msix_vec->ceq_id = ceq_id;
rf->sc_dev.irq_ops->irdma_cfg_ceq(&rf->sc_dev, ceq_id, msix_vec->idx, true);
@@ -1655,10 +1623,7 @@
struct irdma_sc_dev *dev = &rf->sc_dev;
u32 qpcnt;
- if (rf->rdma_ver == IRDMA_GEN_1)
- qpcnt = rsrc_limits_table[rf->limits_sel].qplimit * 2;
- else
- qpcnt = rsrc_limits_table[rf->limits_sel].qplimit;
+ qpcnt = rsrc_limits_table[rf->limits_sel].qplimit;
rf->sd_type = IRDMA_SD_TYPE_DIRECT;
status = irdma_cfg_fpm_val(dev, qpcnt);
@@ -1687,8 +1652,8 @@
if (rf->rdma_ver != IRDMA_GEN_1) {
kfree(rf->allocated_ws_nodes);
rf->allocated_ws_nodes = NULL;
- mutex_destroy(&dev->ws_mutex);
}
+ mutex_destroy(&dev->ws_mutex);
kfree(rf->ceqlist);
rf->ceqlist = NULL;
kfree(rf->iw_msixtbl);
@@ -1696,7 +1661,6 @@
kfree(rf->hmc_info_mem);
rf->hmc_info_mem = NULL;
}
-
/**
* irdma_initialize_dev - initialize device
* @rf: RDMA PCI function
@@ -1746,7 +1710,7 @@
info.bar0 = rf->hw.hw_addr;
info.hmc_fn_id = rf->peer_info->pf_id;
info.hw = &rf->hw;
- status = irdma_sc_dev_init(rf->rdma_ver, &rf->sc_dev, &info);
+ status = irdma_sc_dev_init(&rf->sc_dev, &info);
if (status)
goto error;
@@ -1996,10 +1960,6 @@
rf->rsrc_created = true;
}
- iwdev->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY |
- IB_DEVICE_MEM_WINDOW |
- IB_DEVICE_MEM_MGT_EXTENSIONS;
-
if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
irdma_alloc_set_mac(iwdev);
irdma_add_ip(iwdev);
@@ -2233,28 +2193,20 @@
break;
cqp_request = (struct irdma_cqp_request *)
- (unsigned long)info.scratch;
+ (uintptr_t)info.scratch;
if (info.error && irdma_cqp_crit_err(dev, cqp_request->info.cqp_cmd,
info.maj_err_code,
info.min_err_code))
irdma_dev_err(dev, "cqp opcode = 0x%x maj_err_code = 0x%x min_err_code = 0x%x\n",
info.op_code, info.maj_err_code,
info.min_err_code);
+
if (cqp_request) {
cqp_request->compl_info.maj_err_code = info.maj_err_code;
cqp_request->compl_info.min_err_code = info.min_err_code;
cqp_request->compl_info.op_ret_val = info.op_ret_val;
cqp_request->compl_info.error = info.error;
-
- if (cqp_request->waiting) {
- cqp_request->request_done = true;
- wake_up(&cqp_request->waitq);
- irdma_put_cqp_request(&rf->cqp, cqp_request);
- } else {
- if (cqp_request->callback_fcn)
- cqp_request->callback_fcn(cqp_request);
- irdma_put_cqp_request(&rf->cqp, cqp_request);
- }
+ irdma_complete_cqp_request(&rf->cqp, cqp_request);
}
cqe_count++;
@@ -2545,7 +2497,7 @@
* @action: add, delete or modify
*/
void
-irdma_manage_arp_cache(struct irdma_pci_f *rf, unsigned char *mac_addr,
+irdma_manage_arp_cache(struct irdma_pci_f *rf, const unsigned char *mac_addr,
u32 *ip_addr, u32 action)
{
struct irdma_add_arp_cache_entry_info *info;
@@ -2798,29 +2750,30 @@
info.sq = flush_mask & IRDMA_FLUSH_SQ;
info.rq = flush_mask & IRDMA_FLUSH_RQ;
- if (flush_mask & IRDMA_REFLUSH) {
- if (info.sq)
- iwqp->sc_qp.flush_sq = false;
- if (info.rq)
- iwqp->sc_qp.flush_rq = false;
- }
-
/* Generate userflush errors in CQE */
info.sq_major_code = IRDMA_FLUSH_MAJOR_ERR;
info.sq_minor_code = FLUSH_GENERAL_ERR;
info.rq_major_code = IRDMA_FLUSH_MAJOR_ERR;
info.rq_minor_code = FLUSH_GENERAL_ERR;
info.userflushcode = true;
- if (flush_code) {
- if (info.sq && iwqp->sc_qp.sq_flush_code)
- info.sq_minor_code = flush_code;
- if (info.rq && iwqp->sc_qp.rq_flush_code)
- info.rq_minor_code = flush_code;
- }
- if (irdma_upload_context && !(flush_mask & IRDMA_REFLUSH) &&
- irdma_upload_qp_context(iwqp, 0, 1))
- irdma_print("failed to upload QP context\n");
+ if (flush_mask & IRDMA_REFLUSH) {
+ if (info.sq)
+ iwqp->sc_qp.flush_sq = false;
+ if (info.rq)
+ iwqp->sc_qp.flush_rq = false;
+ } else {
+ if (flush_code) {
+ if (info.sq && iwqp->sc_qp.sq_flush_code)
+ info.sq_minor_code = flush_code;
+ if (info.rq && iwqp->sc_qp.rq_flush_code)
+ info.rq_minor_code = flush_code;
+ }
+ if (irdma_upload_context && irdma_upload_qp_context(iwqp, 0, 1))
+ irdma_print("failed to upload QP context\n");
+ if (!iwqp->user_mode)
+ irdma_sched_qp_flush_work(iwqp);
+ }
/* Issue flush */
(void)irdma_hw_flush_wqes(rf, &iwqp->sc_qp, &info,
diff --git a/sys/dev/irdma/irdma_kcompat.c b/sys/dev/irdma/irdma_kcompat.c
--- a/sys/dev/irdma/irdma_kcompat.c
+++ b/sys/dev/irdma/irdma_kcompat.c
@@ -35,6 +35,36 @@
#include "irdma_main.h"
+#define IRDMA_ROCE_UDP_ENCAP_VALID_PORT_MIN (0xC000)
+
+static u16 kc_rdma_flow_label_to_udp_sport(u32 fl) {
+ u32 fl_low = fl & 0x03FFF;
+ u32 fl_high = fl & 0xFC000;
+
+ fl_low ^= fl_high >> 14;
+
+ return (u16)(fl_low | IRDMA_ROCE_UDP_ENCAP_VALID_PORT_MIN);
+}
+
+#define IRDMA_GRH_FLOWLABEL_MASK (0x000FFFFF)
+
+static u32 kc_rdma_calc_flow_label(u32 lqpn, u32 rqpn) {
+ u64 fl = (u64)lqpn * rqpn;
+
+ fl ^= fl >> 20;
+ fl ^= fl >> 40;
+
+ return (u32)(fl & IRDMA_GRH_FLOWLABEL_MASK);
+}
+
+u16
+kc_rdma_get_udp_sport(u32 fl, u32 lqpn, u32 rqpn)
+{
+ if (!fl)
+ fl = kc_rdma_calc_flow_label(lqpn, rqpn);
+ return kc_rdma_flow_label_to_udp_sport(fl);
+}
+
void
irdma_get_dev_fw_str(struct ib_device *dev,
char *str,
@@ -106,8 +136,10 @@
iwmr->type = IRDMA_MEMREG_TYPE_MEM;
palloc = &iwpbl->pble_alloc;
iwmr->page_cnt = max_num_sg;
+ /* Assume system PAGE_SIZE as the sg page sizes are unknown. */
+ iwmr->len = max_num_sg * PAGE_SIZE;
status = irdma_get_pble(iwdev->rf->pble_rsrc, palloc, iwmr->page_cnt,
- true);
+ false);
if (status)
goto err_get_pble;
@@ -128,6 +160,8 @@
return ERR_PTR(err_code);
}
+#define IRDMA_ALLOC_UCTX_MIN_REQ_LEN offsetofend(struct irdma_alloc_ucontext_req, rsvd8)
+#define IRDMA_ALLOC_UCTX_MIN_RESP_LEN offsetofend(struct irdma_alloc_ucontext_resp, rsvd)
/**
* irdma_alloc_ucontext - Allocate the user context data structure
* @uctx: context
@@ -141,11 +175,15 @@
{
struct ib_device *ibdev = uctx->device;
struct irdma_device *iwdev = to_iwdev(ibdev);
- struct irdma_alloc_ucontext_req req;
+ struct irdma_alloc_ucontext_req req = {0};
struct irdma_alloc_ucontext_resp uresp = {0};
struct irdma_ucontext *ucontext = to_ucontext(uctx);
struct irdma_uk_attrs *uk_attrs;
+ if (udata->inlen < IRDMA_ALLOC_UCTX_MIN_REQ_LEN ||
+ udata->outlen < IRDMA_ALLOC_UCTX_MIN_RESP_LEN)
+ return -EINVAL;
+
if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen)))
return -EINVAL;
@@ -157,7 +195,7 @@
uk_attrs = &iwdev->rf->sc_dev.hw_attrs.uk_attrs;
/* GEN_1 support for libi40iw */
- if (udata->outlen < sizeof(uresp)) {
+ if (udata->outlen == IRDMA_ALLOC_UCTX_MIN_RESP_LEN) {
if (uk_attrs->hw_rev != IRDMA_GEN_1)
return -EOPNOTSUPP;
@@ -169,15 +207,8 @@
if (ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen)))
return -EFAULT;
} else {
- u64 bar_off =
- (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET];
- ucontext->db_mmap_entry =
- irdma_user_mmap_entry_insert(ucontext, bar_off,
- IRDMA_MMAP_IO_NC,
- &uresp.db_mmap_key);
- if (!ucontext->db_mmap_entry) {
- return -ENOMEM;
- }
+ u64 bar_off;
+
uresp.kernel_ver = IRDMA_ABI_VER;
uresp.feature_flags = uk_attrs->feature_flags;
uresp.max_hw_wq_frags = uk_attrs->max_hw_wq_frags;
@@ -189,6 +220,17 @@
uresp.max_hw_cq_size = uk_attrs->max_hw_cq_size;
uresp.min_hw_cq_size = uk_attrs->min_hw_cq_size;
uresp.hw_rev = uk_attrs->hw_rev;
+
+ bar_off =
+ (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET];
+ ucontext->db_mmap_entry =
+ irdma_user_mmap_entry_insert(ucontext, bar_off,
+ IRDMA_MMAP_IO_NC,
+ &uresp.db_mmap_key);
+ if (!ucontext->db_mmap_entry) {
+ return -ENOMEM;
+ }
+
if (ib_copy_to_udata(udata, &uresp,
min(sizeof(uresp), udata->outlen))) {
rdma_user_mmap_entry_remove(ucontext->db_mmap_entry);
@@ -226,6 +268,7 @@
return;
}
+#define IRDMA_ALLOC_PD_MIN_RESP_LEN offsetofend(struct irdma_alloc_pd_resp, rsvd)
/**
* irdma_alloc_pd - allocate protection domain
* @pd: protection domain
@@ -243,6 +286,9 @@
u32 pd_id = 0;
int err;
+ if (udata && udata->outlen < IRDMA_ALLOC_PD_MIN_RESP_LEN)
+ return -EINVAL;
+
err = irdma_alloc_rsrc(rf, rf->allocated_pds, rf->max_pd, &pd_id,
&rf->next_pd);
if (err)
@@ -284,8 +330,7 @@
}
static void
-irdma_fill_ah_info(struct vnet *vnet,
- struct irdma_ah_info *ah_info,
+irdma_fill_ah_info(struct vnet *vnet, struct irdma_ah_info *ah_info,
const struct ib_gid_attr *sgid_attr,
struct sockaddr *sgid_addr, struct sockaddr *dgid_addr,
u8 *dmac, u8 net_type)
@@ -296,9 +341,12 @@
ntohl(((struct sockaddr_in *)dgid_addr)->sin_addr.s_addr);
ah_info->src_ip_addr[0] =
ntohl(((struct sockaddr_in *)sgid_addr)->sin_addr.s_addr);
- ah_info->do_lpbk = irdma_ipv4_is_lpb(vnet,
- ah_info->src_ip_addr[0],
+#ifdef VIMAGE
+ CURVNET_SET_QUIET(vnet);
+ ah_info->do_lpbk = irdma_ipv4_is_lpb(ah_info->src_ip_addr[0],
ah_info->dest_ip_addr[0]);
+ CURVNET_RESTORE();
+#endif
if (ipv4_is_multicast(((struct sockaddr_in *)dgid_addr)->sin_addr.s_addr)) {
irdma_mcast_mac_v4(ah_info->dest_ip_addr, dmac);
}
@@ -337,8 +385,13 @@
if (ah_info->vlan_tag < VLAN_N_VID) {
ah_info->insert_vlan_tag = true;
ah_info->vlan_tag |=
- rt_tos2priority(ah_info->tc_tos) << VLAN_PRIO_SHIFT;
+ (u16)rt_tos2priority(ah_info->tc_tos) << VLAN_PRIO_SHIFT;
}
+ if (iwdev->roce_dcqcn_en) {
+ ah_info->tc_tos &= ~ECN_CODE_PT_MASK;
+ ah_info->tc_tos |= ECN_CODE_PT_VAL;
+ }
+
return 0;
}
@@ -347,7 +400,8 @@
struct irdma_sc_ah *sc_ah, bool sleep)
{
if (!sleep) {
- int cnt = CQP_COMPL_WAIT_TIME_MS * CQP_TIMEOUT_THRESHOLD;
+ int cnt = rf->sc_dev.hw_attrs.max_cqp_compl_wait_time_ms *
+ CQP_TIMEOUT_THRESHOLD;
do {
irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq);
@@ -360,6 +414,8 @@
return 0;
}
+#define IRDMA_CREATE_AH_MIN_RESP_LEN offsetofend(struct irdma_create_ah_resp, rsvd)
+
/**
* irdma_create_ah - create address handle
* @ib_ah: ptr to AH
@@ -391,7 +447,10 @@
} sgid_addr, dgid_addr;
int err;
u8 dmac[ETH_ALEN];
- bool sleep;
+ bool sleep = (flags & RDMA_CREATE_AH_SLEEPABLE) != 0;
+
+ if (udata && udata->outlen < IRDMA_CREATE_AH_MIN_RESP_LEN)
+ return -EINVAL;
err = irdma_alloc_rsrc(rf, rf->allocated_ahs,
rf->max_ah, &ah_id, &rf->next_ah);
@@ -415,7 +474,7 @@
"GID lookup at idx=%d with port=%d failed\n",
attr->grh.sgid_index, attr->port_num);
err = -EINVAL;
- goto error;
+ goto err_gid_l2;
}
rdma_gid2ip((struct sockaddr *)&sgid_addr, &sgid);
rdma_gid2ip((struct sockaddr *)&dgid_addr, &attr->grh.dgid);
@@ -442,38 +501,42 @@
ether_addr_copy(dmac, attr->dmac);
- irdma_fill_ah_info(iwdev->netdev->if_vnet,
- ah_info, &sgid_attr, &sgid_addr.saddr, &dgid_addr.saddr,
+ irdma_fill_ah_info(iwdev->netdev->if_vnet, ah_info, &sgid_attr, &sgid_addr.saddr, &dgid_addr.saddr,
dmac, ah->av.net_type);
err = irdma_create_ah_vlan_tag(iwdev, ah_info, &sgid_attr, dmac);
if (err)
- goto error;
-
- sleep = flags & RDMA_CREATE_AH_SLEEPABLE;
+ goto err_gid_l2;
err = irdma_ah_cqp_op(iwdev->rf, sc_ah, IRDMA_OP_AH_CREATE,
sleep, irdma_gsi_ud_qp_ah_cb, sc_ah);
if (err) {
irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS,
"CQP-OP Create AH fail");
- goto error;
+ goto err_gid_l2;
}
err = irdma_create_ah_wait(rf, sc_ah, sleep);
if (err) {
irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS,
"CQP create AH timed out");
- goto error;
+ goto err_gid_l2;
}
if (udata) {
uresp.ah_id = ah->sc_ah.ah_info.ah_idx;
err = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
+ if (err) {
+ irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah,
+ IRDMA_OP_AH_DESTROY, false, NULL, ah);
+ goto err_gid_l2;
+ }
}
+
return 0;
-error:
+err_gid_l2:
irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, ah_id);
+
return err;
}
@@ -539,35 +602,34 @@
struct ib_qp_init_attr *init_attr,
struct ib_udata *udata)
{
+#define IRDMA_CREATE_QP_MIN_REQ_LEN offsetofend(struct irdma_create_qp_req, user_compl_ctx)
+#define IRDMA_CREATE_QP_MIN_RESP_LEN offsetofend(struct irdma_create_qp_resp, rsvd)
struct irdma_pd *iwpd = to_iwpd(ibpd);
struct irdma_device *iwdev = to_iwdev(ibpd->device);
struct irdma_pci_f *rf = iwdev->rf;
struct irdma_qp *iwqp;
- struct irdma_create_qp_req req;
struct irdma_create_qp_resp uresp = {0};
u32 qp_num = 0;
int ret;
int err_code;
- int sq_size;
- int rq_size;
struct irdma_sc_qp *qp;
struct irdma_sc_dev *dev = &rf->sc_dev;
struct irdma_uk_attrs *uk_attrs = &dev->hw_attrs.uk_attrs;
struct irdma_qp_init_info init_info = {{0}};
struct irdma_qp_host_ctx_info *ctx_info;
- unsigned long flags;
err_code = irdma_validate_qp_attrs(init_attr, iwdev);
if (err_code)
return ERR_PTR(err_code);
- sq_size = init_attr->cap.max_send_wr;
- rq_size = init_attr->cap.max_recv_wr;
+ if (udata && (udata->inlen < IRDMA_CREATE_QP_MIN_REQ_LEN ||
+ udata->outlen < IRDMA_CREATE_QP_MIN_RESP_LEN))
+ return ERR_PTR(-EINVAL);
init_info.vsi = &iwdev->vsi;
init_info.qp_uk_init_info.uk_attrs = uk_attrs;
- init_info.qp_uk_init_info.sq_size = sq_size;
- init_info.qp_uk_init_info.rq_size = rq_size;
+ init_info.qp_uk_init_info.sq_size = init_attr->cap.max_send_wr;
+ init_info.qp_uk_init_info.rq_size = init_attr->cap.max_recv_wr;
init_info.qp_uk_init_info.max_sq_frag_cnt = init_attr->cap.max_send_sge;
init_info.qp_uk_init_info.max_rq_frag_cnt = init_attr->cap.max_recv_sge;
init_info.qp_uk_init_info.max_inline_data = init_attr->cap.max_inline_data;
@@ -630,35 +692,8 @@
init_waitqueue_head(&iwqp->mod_qp_waitq);
if (udata) {
- err_code = ib_copy_from_udata(&req, udata,
- min(sizeof(req), udata->inlen));
- if (err_code) {
- irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS,
- "ib_copy_from_data fail\n");
- goto error;
- }
-
- iwqp->ctx_info.qp_compl_ctx = req.user_compl_ctx;
- iwqp->user_mode = 1;
- if (req.user_wqe_bufs) {
- struct irdma_ucontext *ucontext = to_ucontext(ibpd->uobject->context);
-
- init_info.qp_uk_init_info.legacy_mode = ucontext->legacy_mode;
- spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags);
- iwqp->iwpbl = irdma_get_pbl((unsigned long)req.user_wqe_bufs,
- &ucontext->qp_reg_mem_list);
- spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags);
-
- if (!iwqp->iwpbl) {
- err_code = -ENODATA;
- irdma_debug(iwdev_to_idev(iwdev),
- IRDMA_DEBUG_VERBS,
- "no pbl info\n");
- goto error;
- }
- }
init_info.qp_uk_init_info.abi_ver = iwpd->sc_pd.abi_ver;
- irdma_setup_virt_qp(iwdev, iwqp, &init_info);
+ err_code = irdma_setup_umode_qp(udata, iwdev, iwqp, &init_info, init_attr);
} else {
INIT_DELAYED_WORK(&iwqp->dwork_flush, irdma_flush_worker);
init_info.qp_uk_init_info.abi_ver = IRDMA_ABI_VER;
@@ -713,8 +748,6 @@
spin_lock_init(&iwqp->sc_qp.pfpdu.lock);
iwqp->sig_all = (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ? 1 : 0;
rf->qp_table[qp_num] = iwqp;
- iwqp->max_send_wr = sq_size;
- iwqp->max_recv_wr = rq_size;
if (rdma_protocol_roce(&iwdev->ibdev, 1)) {
if (dev->ws_add(&iwdev->vsi, 0)) {
@@ -735,8 +768,8 @@
if (rdma_protocol_iwarp(&iwdev->ibdev, 1))
uresp.lsmm = 1;
}
- uresp.actual_sq_size = sq_size;
- uresp.actual_rq_size = rq_size;
+ uresp.actual_sq_size = init_info.qp_uk_init_info.sq_size;
+ uresp.actual_rq_size = init_info.qp_uk_init_info.rq_size;
uresp.qp_id = qp_num;
uresp.qp_caps = qp->qp_uk.qp_caps;
@@ -776,9 +809,6 @@
if (iwqp->iwarp_state == IRDMA_QP_STATE_RTS)
irdma_modify_qp_to_err(&iwqp->sc_qp);
- if (!iwqp->user_mode)
- cancel_delayed_work_sync(&iwqp->dwork_flush);
-
irdma_qp_rem_ref(&iwqp->ibqp);
wait_for_completion(&iwqp->free_qp);
irdma_free_lsmm_rsrc(iwqp);
@@ -810,6 +840,8 @@
const struct ib_cq_init_attr *attr,
struct ib_udata *udata)
{
+#define IRDMA_CREATE_CQ_MIN_REQ_LEN offsetofend(struct irdma_create_cq_req, user_cq_buf)
+#define IRDMA_CREATE_CQ_MIN_RESP_LEN offsetofend(struct irdma_create_cq_resp, cq_size)
struct ib_device *ibdev = ibcq->device;
struct irdma_device *iwdev = to_iwdev(ibdev);
struct irdma_pci_f *rf = iwdev->rf;
@@ -825,10 +857,15 @@
unsigned long flags;
int err_code;
int entries = attr->cqe;
+ bool cqe_64byte_ena;
err_code = cq_validate_flags(attr->flags, dev->hw_attrs.uk_attrs.hw_rev);
if (err_code)
return err_code;
+
+ if (udata && (udata->inlen < IRDMA_CREATE_CQ_MIN_REQ_LEN ||
+ udata->outlen < IRDMA_CREATE_CQ_MIN_RESP_LEN))
+ return -EINVAL;
err_code = irdma_alloc_rsrc(rf, rf->allocated_cqs, rf->max_cq, &cq_num,
&rf->next_cq);
if (err_code)
@@ -842,6 +879,8 @@
info.dev = dev;
ukinfo->cq_size = max(entries, 4);
ukinfo->cq_id = cq_num;
+ cqe_64byte_ena = (dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_64_BYTE_CQE) ? true : false;
+ ukinfo->avoid_mem_cflct = cqe_64byte_ena;
iwcq->ibcq.cqe = info.cq_uk_init_info.cq_size;
if (attr->comp_vector < rf->ceqs_count)
info.ceq_id = attr->comp_vector;
@@ -860,6 +899,7 @@
iwcq->user_mode = true;
ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext);
+
if (ib_copy_from_udata(&req, udata,
min(sizeof(req), udata->inlen))) {
err_code = -EFAULT;
@@ -913,14 +953,17 @@
}
entries++;
- if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
+ if (!cqe_64byte_ena && dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
entries *= 2;
ukinfo->cq_size = entries;
- rsize = info.cq_uk_init_info.cq_size * sizeof(struct irdma_cqe);
- iwcq->kmem.size = round_up(rsize, 256);
+ if (cqe_64byte_ena)
+ rsize = info.cq_uk_init_info.cq_size * sizeof(struct irdma_extended_cqe);
+ else
+ rsize = info.cq_uk_init_info.cq_size * sizeof(struct irdma_cqe);
+ iwcq->kmem.size = round_up(rsize, IRDMA_HW_PAGE_SIZE);
iwcq->kmem.va = irdma_allocate_dma_mem(dev->hw, &iwcq->kmem,
- iwcq->kmem.size, 256);
+ iwcq->kmem.size, IRDMA_HW_PAGE_SIZE);
if (!iwcq->kmem.va) {
err_code = -ENOMEM;
goto cq_free_rsrc;
@@ -1058,61 +1101,97 @@
int
irdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
{
- struct ib_pd *ibpd = ib_mr->pd;
- struct irdma_pd *iwpd = to_iwpd(ibpd);
struct irdma_mr *iwmr = to_iwmr(ib_mr);
struct irdma_device *iwdev = to_iwdev(ib_mr->device);
- struct irdma_dealloc_stag_info *info;
struct irdma_pbl *iwpbl = &iwmr->iwpbl;
- struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc;
- struct irdma_cqp_request *cqp_request;
- struct cqp_cmds_info *cqp_info;
- int status;
+ int ret;
if (iwmr->type != IRDMA_MEMREG_TYPE_MEM) {
if (iwmr->region) {
struct irdma_ucontext *ucontext;
- ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext);
+ ucontext = rdma_udata_to_drv_context(udata,
+ struct irdma_ucontext,
+ ibucontext);
irdma_del_memlist(iwmr, ucontext);
}
goto done;
}
- cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true);
- if (!cqp_request)
- return -ENOMEM;
-
- cqp_info = &cqp_request->info;
- info = &cqp_info->in.u.dealloc_stag.info;
- memset(info, 0, sizeof(*info));
- info->pd_id = iwpd->sc_pd.pd_id & 0x00007fff;
- info->stag_idx = RS_64_1(ib_mr->rkey, IRDMA_CQPSQ_STAG_IDX_S);
- info->mr = true;
- if (iwpbl->pbl_allocated)
- info->dealloc_pbl = true;
-
- cqp_info->cqp_cmd = IRDMA_OP_DEALLOC_STAG;
- cqp_info->post_sq = 1;
- cqp_info->in.u.dealloc_stag.dev = &iwdev->rf->sc_dev;
- cqp_info->in.u.dealloc_stag.scratch = (uintptr_t)cqp_request;
- status = irdma_handle_cqp_op(iwdev->rf, cqp_request);
- irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request);
- if (status)
- return status;
+ ret = irdma_hwdereg_mr(ib_mr);
+ if (ret)
+ return ret;
irdma_free_stag(iwdev, iwmr->stag);
done:
if (iwpbl->pbl_allocated)
- irdma_free_pble(iwdev->rf->pble_rsrc, palloc);
+ irdma_free_pble(iwdev->rf->pble_rsrc, &iwpbl->pble_alloc);
if (iwmr->region)
ib_umem_release(iwmr->region);
+
kfree(iwmr);
return 0;
}
+/*
+ * irdma_rereg_user_mr - Re-Register a user memory region @ibmr: ib mem to access iwarp mr pointer @flags: bit mask to
+ * indicate which of the attr's of MR modified @start: virtual start address @len: length of mr @virt: virtual address
+ * @new access flags: bit mask of access flags @new_pd: ptr of pd @udata: user data
+ */
+int
+irdma_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, u64 len,
+ u64 virt, int new_access, struct ib_pd *new_pd,
+ struct ib_udata *udata)
+{
+ struct irdma_device *iwdev = to_iwdev(ib_mr->device);
+ struct irdma_mr *iwmr = to_iwmr(ib_mr);
+ struct irdma_pbl *iwpbl = &iwmr->iwpbl;
+ int ret;
+
+ if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size)
+ return -EINVAL;
+
+ if (flags & ~(IB_MR_REREG_TRANS | IB_MR_REREG_PD | IB_MR_REREG_ACCESS))
+ return -EOPNOTSUPP;
+
+ ret = irdma_hwdereg_mr(ib_mr);
+ if (ret)
+ return ret;
+
+ if (flags & IB_MR_REREG_ACCESS)
+ iwmr->access = new_access;
+
+ if (flags & IB_MR_REREG_PD) {
+ iwmr->ibmr.pd = new_pd;
+ iwmr->ibmr.device = new_pd->device;
+ }
+
+ if (flags & IB_MR_REREG_TRANS) {
+ if (iwpbl->pbl_allocated) {
+ irdma_free_pble(iwdev->rf->pble_rsrc,
+ &iwpbl->pble_alloc);
+ iwpbl->pbl_allocated = false;
+ }
+ if (iwmr->region) {
+ ib_umem_release(iwmr->region);
+ iwmr->region = NULL;
+ }
+
+ ib_mr = irdma_rereg_mr_trans(iwmr, start, len, virt, udata);
+ if (IS_ERR(ib_mr))
+ return PTR_ERR(ib_mr);
+
+ } else {
+ ret = irdma_hwreg_mr(iwdev, iwmr, iwmr->access);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+
int
kc_irdma_set_roce_cm_info(struct irdma_qp *iwqp, struct ib_qp_attr *attr,
u16 *vlan_id)
@@ -1134,8 +1213,8 @@
}
rdma_gid2ip((struct sockaddr *)&av->sgid_addr, &sgid);
-
dev_put(sgid_attr.ndev);
+ iwqp->sc_qp.user_pri = iwqp->ctx_info.user_pri;
return 0;
}
@@ -1167,11 +1246,11 @@
wait_for_completion(&iwcq->free_cq);
irdma_cq_wq_destroy(iwdev->rf, cq);
- irdma_cq_free_rsrc(iwdev->rf, iwcq);
spin_lock_irqsave(&iwceq->ce_lock, flags);
irdma_sc_cleanup_ceqes(cq, ceq);
spin_unlock_irqrestore(&iwceq->ce_lock, flags);
+ irdma_cq_free_rsrc(iwdev->rf, iwcq);
}
/**
@@ -1441,7 +1520,59 @@
return 0;
}
-extern const char *const irdma_hw_stat_names[];
+static const char *const irdma_hw_stat_names[] = {
+ /* gen1 - 32-bit */
+ [IRDMA_HW_STAT_INDEX_IP4RXDISCARD] = "ip4InDiscards",
+ [IRDMA_HW_STAT_INDEX_IP4RXTRUNC] = "ip4InTruncatedPkts",
+ [IRDMA_HW_STAT_INDEX_IP4TXNOROUTE] = "ip4OutNoRoutes",
+ [IRDMA_HW_STAT_INDEX_IP6RXDISCARD] = "ip6InDiscards",
+ [IRDMA_HW_STAT_INDEX_IP6RXTRUNC] = "ip6InTruncatedPkts",
+ [IRDMA_HW_STAT_INDEX_IP6TXNOROUTE] = "ip6OutNoRoutes",
+ [IRDMA_HW_STAT_INDEX_RXVLANERR] = "rxVlanErrors",
+ /* gen1 - 64-bit */
+ [IRDMA_HW_STAT_INDEX_IP4RXOCTS] = "ip4InOctets",
+ [IRDMA_HW_STAT_INDEX_IP4RXPKTS] = "ip4InPkts",
+ [IRDMA_HW_STAT_INDEX_IP4RXFRAGS] = "ip4InReasmRqd",
+ [IRDMA_HW_STAT_INDEX_IP4RXMCPKTS] = "ip4InMcastPkts",
+ [IRDMA_HW_STAT_INDEX_IP4TXOCTS] = "ip4OutOctets",
+ [IRDMA_HW_STAT_INDEX_IP4TXPKTS] = "ip4OutPkts",
+ [IRDMA_HW_STAT_INDEX_IP4TXFRAGS] = "ip4OutSegRqd",
+ [IRDMA_HW_STAT_INDEX_IP4TXMCPKTS] = "ip4OutMcastPkts",
+ [IRDMA_HW_STAT_INDEX_IP6RXOCTS] = "ip6InOctets",
+ [IRDMA_HW_STAT_INDEX_IP6RXPKTS] = "ip6InPkts",
+ [IRDMA_HW_STAT_INDEX_IP6RXFRAGS] = "ip6InReasmRqd",
+ [IRDMA_HW_STAT_INDEX_IP6RXMCPKTS] = "ip6InMcastPkts",
+ [IRDMA_HW_STAT_INDEX_IP6TXOCTS] = "ip6OutOctets",
+ [IRDMA_HW_STAT_INDEX_IP6TXPKTS] = "ip6OutPkts",
+ [IRDMA_HW_STAT_INDEX_IP6TXFRAGS] = "ip6OutSegRqd",
+ [IRDMA_HW_STAT_INDEX_IP6TXMCPKTS] = "ip6OutMcastPkts",
+ [IRDMA_HW_STAT_INDEX_RDMARXRDS] = "InRdmaReads",
+ [IRDMA_HW_STAT_INDEX_RDMARXSNDS] = "InRdmaSends",
+ [IRDMA_HW_STAT_INDEX_RDMARXWRS] = "InRdmaWrites",
+ [IRDMA_HW_STAT_INDEX_RDMATXRDS] = "OutRdmaReads",
+ [IRDMA_HW_STAT_INDEX_RDMATXSNDS] = "OutRdmaSends",
+ [IRDMA_HW_STAT_INDEX_RDMATXWRS] = "OutRdmaWrites",
+ [IRDMA_HW_STAT_INDEX_RDMAVBND] = "RdmaBnd",
+ [IRDMA_HW_STAT_INDEX_RDMAVINV] = "RdmaInv",
+
+ /* gen2 - 32-bit */
+ [IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED] = "cnpHandled",
+ [IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED] = "cnpIgnored",
+ [IRDMA_HW_STAT_INDEX_TXNPCNPSENT] = "cnpSent",
+ /* gen2 - 64-bit */
+ [IRDMA_HW_STAT_INDEX_IP4RXMCOCTS] = "ip4InMcastOctets",
+ [IRDMA_HW_STAT_INDEX_IP4TXMCOCTS] = "ip4OutMcastOctets",
+ [IRDMA_HW_STAT_INDEX_IP6RXMCOCTS] = "ip6InMcastOctets",
+ [IRDMA_HW_STAT_INDEX_IP6TXMCOCTS] = "ip6OutMcastOctets",
+ [IRDMA_HW_STAT_INDEX_UDPRXPKTS] = "RxUDP",
+ [IRDMA_HW_STAT_INDEX_UDPTXPKTS] = "TxUDP",
+ [IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS] = "RxECNMrkd",
+ [IRDMA_HW_STAT_INDEX_TCPRTXSEG] = "RetransSegs",
+ [IRDMA_HW_STAT_INDEX_TCPRXOPTERR] = "InOptErrors",
+ [IRDMA_HW_STAT_INDEX_TCPRXPROTOERR] = "InProtoErrors",
+ [IRDMA_HW_STAT_INDEX_TCPRXSEGS] = "InSegs",
+ [IRDMA_HW_STAT_INDEX_TCPTXSEG] = "OutSegs",
+};
/**
* irdma_alloc_hw_stats - Allocate a hw stats structure
@@ -1546,6 +1677,7 @@
BIT_ULL(IB_USER_VERBS_CMD_ALLOC_PD) |
BIT_ULL(IB_USER_VERBS_CMD_DEALLOC_PD) |
BIT_ULL(IB_USER_VERBS_CMD_REG_MR) |
+ BIT_ULL(IB_USER_VERBS_CMD_REREG_MR) |
BIT_ULL(IB_USER_VERBS_CMD_DEREG_MR) |
BIT_ULL(IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
BIT_ULL(IB_USER_VERBS_CMD_CREATE_CQ) |
diff --git a/sys/dev/irdma/irdma_main.h b/sys/dev/irdma/irdma_main.h
--- a/sys/dev/irdma/irdma_main.h
+++ b/sys/dev/irdma/irdma_main.h
@@ -44,6 +44,7 @@
#include <netinet/if_ether.h>
#include <linux/slab.h>
#include <linux/rculist.h>
+#include <rdma/uverbs_ioctl.h>
#include <rdma/ib_smi.h>
#include <rdma/ib_verbs.h>
#include <rdma/ib_pack.h>
@@ -52,7 +53,6 @@
#include <rdma/ib_user_verbs.h>
#include <rdma/ib_umem.h>
#include <rdma/ib_cache.h>
-#include <rdma/uverbs_ioctl.h>
#include "osdep.h"
#include "irdma_defs.h"
#include "irdma_hmc.h"
@@ -101,7 +101,7 @@
#define IRDMA_NO_QSET 0xffff
#define IW_CFG_FPM_QP_COUNT 32768
-#define IRDMA_MAX_PAGES_PER_FMR 512
+#define IRDMA_MAX_PAGES_PER_FMR 262144
#define IRDMA_MIN_PAGES_PER_FMR 1
#define IRDMA_CQP_COMPL_RQ_WQE_FLUSHED 2
#define IRDMA_CQP_COMPL_SQ_WQE_FLUSHED 3
@@ -123,9 +123,6 @@
#define IRDMA_DRV_OPT_MCAST_LOGPORT_MAP 0x00000800
#define IW_HMC_OBJ_TYPE_NUM ARRAY_SIZE(iw_hmc_obj_types)
-#define VSI_RXSWCTRL(_VSI) (0x00205000 + ((_VSI) * 4))
-#define VSI_RXSWCTRL_MACVSIPRUNEENABLE_M BIT(8)
-#define VSI_RXSWCTRL_SRCPRUNEENABLE_M BIT(13)
#define IRDMA_ROCE_CWND_DEFAULT 0x400
#define IRDMA_ROCE_ACKCREDS_DEFAULT 0x1E
@@ -278,6 +275,8 @@
u8 *mem_rsrc;
u8 rdma_ver;
u8 rst_to;
+ /* Not used in SRIOV VF mode */
+ u8 pf_id;
enum irdma_protocol_used protocol_used;
bool en_rem_endpoint_trk:1;
bool dcqcn_ena:1;
@@ -360,6 +359,7 @@
struct ib_device ibdev;
struct irdma_pci_f *rf;
struct ifnet *netdev;
+ struct notifier_block nb_netdevice_event;
struct irdma_handler *hdl;
struct workqueue_struct *cleanup_wq;
struct irdma_sc_vsi vsi;
@@ -368,7 +368,6 @@
u32 roce_ackcreds;
u32 vendor_id;
u32 vendor_part_id;
- u32 device_cap_flags;
u32 push_mode;
u32 rcv_wnd;
u16 mac_ip_table_idx;
@@ -376,6 +375,12 @@
u8 rcv_wscale;
u8 iw_status;
u8 rd_fence_rate;
+ bool override_rcv_wnd:1;
+ bool override_cwnd:1;
+ bool override_ackcreds:1;
+ bool override_ooo:1;
+ bool override_rd_fence_rate:1;
+ bool override_rtomin:1;
bool roce_mode:1;
bool roce_dcqcn_en:1;
bool dcb_vlan_mode:1;
@@ -508,7 +513,7 @@
void irdma_free_lsmm_rsrc(struct irdma_qp *iwqp);
struct ib_qp *irdma_get_qp(struct ib_device *ibdev, int qpn);
void irdma_flush_wqes(struct irdma_qp *iwqp, u32 flush_mask);
-void irdma_manage_arp_cache(struct irdma_pci_f *rf, unsigned char *mac_addr,
+void irdma_manage_arp_cache(struct irdma_pci_f *rf, const unsigned char *mac_addr,
u32 *ip_addr, u32 action);
struct irdma_apbvt_entry *irdma_add_apbvt(struct irdma_device *iwdev, u16 port);
void irdma_del_apbvt(struct irdma_device *iwdev,
@@ -581,6 +586,10 @@
void *cb_param);
void irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request *cqp_request);
bool irdma_cq_empty(struct irdma_cq *iwcq);
+int irdma_netdevice_event(struct notifier_block *notifier, unsigned long event,
+ void *ptr);
+void irdma_unregister_notifiers(struct irdma_device *iwdev);
+int irdma_register_notifiers(struct irdma_device *iwdev);
void irdma_set_rf_user_cfg_params(struct irdma_pci_f *rf);
void irdma_add_ip(struct irdma_device *iwdev);
void irdma_add_handler(struct irdma_handler *hdl);
diff --git a/sys/dev/irdma/irdma_pble.c b/sys/dev/irdma/irdma_pble.c
--- a/sys/dev/irdma/irdma_pble.c
+++ b/sys/dev/irdma/irdma_pble.c
@@ -265,7 +265,7 @@
return -EINVAL;
chunkmem.size = sizeof(*chunk);
- chunkmem.va = kzalloc(chunkmem.size, GFP_ATOMIC);
+ chunkmem.va = kzalloc(chunkmem.size, GFP_KERNEL);
if (!chunkmem.va)
return -ENOMEM;
@@ -394,7 +394,7 @@
lvl2->leaf_cnt = total;
lvl2->leafmem.size = (sizeof(*leaf) * total);
- lvl2->leafmem.va = kzalloc(lvl2->leafmem.size, GFP_ATOMIC);
+ lvl2->leafmem.va = kzalloc(lvl2->leafmem.size, GFP_KERNEL);
if (!lvl2->leafmem.va)
return -ENOMEM;
diff --git a/sys/dev/irdma/irdma_protos.h b/sys/dev/irdma/irdma_protos.h
--- a/sys/dev/irdma/irdma_protos.h
+++ b/sys/dev/irdma/irdma_protos.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2016 - 2021 Intel Corporation
+ * Copyright (c) 2016 - 2022 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -45,8 +45,7 @@
#define CQP_TIMEOUT_THRESHOLD 500
/* init operations */
-int irdma_sc_dev_init(enum irdma_vers ver, struct irdma_sc_dev *dev,
- struct irdma_device_init_info *info);
+int irdma_sc_dev_init(struct irdma_sc_dev *dev, struct irdma_device_init_info *info);
void irdma_sc_cqp_post_sq(struct irdma_sc_cqp *cqp);
__le64 *irdma_sc_cqp_get_next_send_wqe(struct irdma_sc_cqp *cqp, u64 scratch);
int irdma_sc_mr_fast_register(struct irdma_sc_qp *qp,
@@ -56,7 +55,7 @@
u8 traffic_class,
u16 qs_handle);
/* HMC/FPM functions */
-int irdma_sc_init_iw_hmc(struct irdma_sc_dev *dev, u8 hmc_fn_id);
+int irdma_sc_init_iw_hmc(struct irdma_sc_dev *dev, u16 hmc_fn_id);
/* stats misc */
int irdma_cqp_gather_stats_cmd(struct irdma_sc_dev *dev,
struct irdma_vsi_pestat *pestat, bool wait);
@@ -114,7 +113,7 @@
u8 irdma_get_encoded_wqe_size(u32 wqsize, enum irdma_queue_type queue_type);
void irdma_modify_qp_to_err(struct irdma_sc_qp *sc_qp);
int irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp *cqp, u64 scratch,
- u8 hmc_fn_id, bool post_sq,
+ u16 hmc_fn_id, bool post_sq,
bool poll_registers);
int irdma_cfg_fpm_val(struct irdma_sc_dev *dev, u32 qp_count);
int irdma_get_rdma_features(struct irdma_sc_dev *dev);
@@ -129,9 +128,9 @@
int irdma_cqp_sds_cmd(struct irdma_sc_dev *dev,
struct irdma_update_sds_info *info);
int irdma_cqp_query_fpm_val_cmd(struct irdma_sc_dev *dev,
- struct irdma_dma_mem *val_mem, u8 hmc_fn_id);
+ struct irdma_dma_mem *val_mem, u16 hmc_fn_id);
int irdma_cqp_commit_fpm_val_cmd(struct irdma_sc_dev *dev,
- struct irdma_dma_mem *val_mem, u8 hmc_fn_id);
+ struct irdma_dma_mem *val_mem, u16 hmc_fn_id);
int irdma_alloc_query_fpm_buf(struct irdma_sc_dev *dev,
struct irdma_dma_mem *mem);
int irdma_cqp_manage_hmc_fcn_cmd(struct irdma_sc_dev *dev,
diff --git a/sys/dev/irdma/irdma_puda.c b/sys/dev/irdma/irdma_puda.c
--- a/sys/dev/irdma/irdma_puda.c
+++ b/sys/dev/irdma/irdma_puda.c
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2021 Intel Corporation
+ * Copyright (c) 2015 - 2022 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -129,17 +129,17 @@
if (!initial)
get_64bit_val(wqe, IRDMA_BYTE_24, &offset24);
- offset24 = (offset24) ? 0 : LS_64(1, IRDMAQPSQ_VALID);
+ offset24 = (offset24) ? 0 : FIELD_PREP(IRDMAQPSQ_VALID, 1);
set_64bit_val(wqe, IRDMA_BYTE_16, 0);
set_64bit_val(wqe, 0, buf->mem.pa);
if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) {
set_64bit_val(wqe, IRDMA_BYTE_8,
- LS_64(buf->mem.size, IRDMAQPSQ_GEN1_FRAG_LEN));
+ FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, buf->mem.size));
} else {
set_64bit_val(wqe, IRDMA_BYTE_8,
- LS_64(buf->mem.size,
- IRDMAQPSQ_FRAG_LEN) | (offset24 & IRDMAQPSQ_VALID_M));
+ FIELD_PREP(IRDMAQPSQ_FRAG_LEN, buf->mem.size) |
+ offset24);
}
irdma_wmb(); /* make sure WQE is written before valid bit is set */
@@ -183,7 +183,7 @@
struct irdma_virt_mem buf_mem;
buf_mem.size = sizeof(struct irdma_puda_buf);
- buf_mem.va = kzalloc(buf_mem.size, GFP_ATOMIC);
+ buf_mem.va = kzalloc(buf_mem.size, GFP_KERNEL);
if (!buf_mem.va)
return NULL;
@@ -269,18 +269,18 @@
cqe = IRDMA_GET_CURRENT_CQ_ELEM(&cq->cq_uk);
get_64bit_val(cqe, IRDMA_BYTE_24, &qword3);
- valid_bit = (bool)RS_64(qword3, IRDMA_CQ_VALID);
+ valid_bit = (bool)FIELD_GET(IRDMA_CQ_VALID, qword3);
if (valid_bit != cq_uk->polarity)
return -ENOENT;
if (cq->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
- ext_valid = (bool)RS_64(qword3, IRDMA_CQ_EXTCQE);
+ ext_valid = (bool)FIELD_GET(IRDMA_CQ_EXTCQE, qword3);
if (ext_valid) {
peek_head = (cq_uk->cq_ring.head + 1) % cq_uk->cq_ring.size;
ext_cqe = cq_uk->cq_base[peek_head].buf;
get_64bit_val(ext_cqe, IRDMA_BYTE_24, &qword7);
- polarity = (u8)RS_64(qword7, IRDMA_CQ_VALID);
+ polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword7);
if (!peek_head)
polarity ^= 1;
if (polarity != cq_uk->polarity)
@@ -298,11 +298,11 @@
irdma_debug_buf(cq->dev, IRDMA_DEBUG_PUDA, "PUDA EXT-CQE",
ext_cqe, 32);
- error = (bool)RS_64(qword3, IRDMA_CQ_ERROR);
+ error = (bool)FIELD_GET(IRDMA_CQ_ERROR, qword3);
if (error) {
irdma_debug(cq->dev, IRDMA_DEBUG_PUDA, "receive error\n");
- major_err = (u32)(RS_64(qword3, IRDMA_CQ_MAJERR));
- minor_err = (u32)(RS_64(qword3, IRDMA_CQ_MINERR));
+ major_err = (u32)(FIELD_GET(IRDMA_CQ_MAJERR, qword3));
+ minor_err = (u32)(FIELD_GET(IRDMA_CQ_MINERR, qword3));
info->compl_error = major_err << 16 | minor_err;
return -EIO;
}
@@ -310,23 +310,23 @@
get_64bit_val(cqe, IRDMA_BYTE_0, &qword0);
get_64bit_val(cqe, IRDMA_BYTE_16, &qword2);
- info->q_type = (u8)RS_64(qword3, IRDMA_CQ_SQ);
- info->qp_id = (u32)RS_64(qword2, IRDMACQ_QPID);
+ info->q_type = (u8)FIELD_GET(IRDMA_CQ_SQ, qword3);
+ info->qp_id = (u32)FIELD_GET(IRDMACQ_QPID, qword2);
if (cq->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
- info->ipv4 = (bool)RS_64(qword3, IRDMACQ_IPV4);
+ info->ipv4 = (bool)FIELD_GET(IRDMACQ_IPV4, qword3);
get_64bit_val(cqe, IRDMA_BYTE_8, &comp_ctx);
info->qp = (struct irdma_qp_uk *)(irdma_uintptr) comp_ctx;
- info->wqe_idx = (u32)RS_64(qword3, IRDMA_CQ_WQEIDX);
+ info->wqe_idx = (u32)FIELD_GET(IRDMA_CQ_WQEIDX, qword3);
if (info->q_type == IRDMA_CQE_QTYPE_RQ) {
if (ext_valid) {
- info->vlan_valid = (bool)RS_64(qword7, IRDMA_CQ_UDVLANVALID);
+ info->vlan_valid = (bool)FIELD_GET(IRDMA_CQ_UDVLANVALID, qword7);
if (info->vlan_valid) {
get_64bit_val(ext_cqe, IRDMA_BYTE_16, &qword6);
- info->vlan = (u16)RS_64(qword6, IRDMA_CQ_UDVLAN);
+ info->vlan = (u16)FIELD_GET(IRDMA_CQ_UDVLAN, qword6);
}
- info->smac_valid = (bool)RS_64(qword7, IRDMA_CQ_UDSMACVALID);
+ info->smac_valid = (bool)FIELD_GET(IRDMA_CQ_UDSMACVALID, qword7);
if (info->smac_valid) {
get_64bit_val(ext_cqe, IRDMA_BYTE_16, &qword6);
info->smac[0] = (u8)((qword6 >> 40) & 0xFF);
@@ -339,12 +339,12 @@
}
if (cq->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) {
- info->vlan_valid = (bool)RS_64(qword3, IRDMA_VLAN_TAG_VALID);
- info->l4proto = (u8)RS_64(qword2, IRDMA_UDA_L4PROTO);
- info->l3proto = (u8)RS_64(qword2, IRDMA_UDA_L3PROTO);
+ info->vlan_valid = (bool)FIELD_GET(IRDMA_VLAN_TAG_VALID, qword3);
+ info->l4proto = (u8)FIELD_GET(IRDMA_UDA_L4PROTO, qword2);
+ info->l3proto = (u8)FIELD_GET(IRDMA_UDA_L3PROTO, qword2);
}
- info->payload_len = (u32)RS_64(qword0, IRDMACQ_PAYLDLEN);
+ info->payload_len = (u32)FIELD_GET(IRDMACQ_PAYLDLEN, qword0);
}
return 0;
@@ -486,35 +486,36 @@
if (qp->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
hdr[0] = 0; /* Dest_QPN and Dest_QKey only for UD */
- hdr[1] = LS_64(IRDMA_OP_TYPE_SEND, IRDMA_UDA_QPSQ_OPCODE) |
- LS_64(l4len, IRDMA_UDA_QPSQ_L4LEN) |
- LS_64(info->ah_id, IRDMAQPSQ_AHID) |
- LS_64(1, IRDMA_UDA_QPSQ_SIGCOMPL) |
- LS_64(qp->qp_uk.swqe_polarity, IRDMA_UDA_QPSQ_VALID);
+ hdr[1] = FIELD_PREP(IRDMA_UDA_QPSQ_OPCODE, IRDMA_OP_TYPE_SEND) |
+ FIELD_PREP(IRDMA_UDA_QPSQ_L4LEN, l4len) |
+ FIELD_PREP(IRDMAQPSQ_AHID, info->ah_id) |
+ FIELD_PREP(IRDMA_UDA_QPSQ_SIGCOMPL, 1) |
+ FIELD_PREP(IRDMA_UDA_QPSQ_VALID,
+ qp->qp_uk.swqe_polarity);
/* Forth line of WQE descriptor */
set_64bit_val(wqe, IRDMA_BYTE_0, info->paddr);
set_64bit_val(wqe, IRDMA_BYTE_8,
- LS_64(info->len, IRDMAQPSQ_FRAG_LEN) |
- LS_64(qp->qp_uk.swqe_polarity, IRDMA_UDA_QPSQ_VALID));
+ FIELD_PREP(IRDMAQPSQ_FRAG_LEN, info->len) |
+ FIELD_PREP(IRDMA_UDA_QPSQ_VALID, qp->qp_uk.swqe_polarity));
} else {
- hdr[0] = LS_64((info->maclen >> 1), IRDMA_UDA_QPSQ_MACLEN) |
- LS_64(iplen, IRDMA_UDA_QPSQ_IPLEN) |
- LS_64(1, IRDMA_UDA_QPSQ_L4T) |
- LS_64(iipt, IRDMA_UDA_QPSQ_IIPT) |
- LS_64(l4len, IRDMA_GEN1_UDA_QPSQ_L4LEN);
+ hdr[0] = FIELD_PREP(IRDMA_UDA_QPSQ_MACLEN, info->maclen >> 1) |
+ FIELD_PREP(IRDMA_UDA_QPSQ_IPLEN, iplen) |
+ FIELD_PREP(IRDMA_UDA_QPSQ_L4T, 1) |
+ FIELD_PREP(IRDMA_UDA_QPSQ_IIPT, iipt) |
+ FIELD_PREP(IRDMA_GEN1_UDA_QPSQ_L4LEN, l4len);
- hdr[1] = LS_64(IRDMA_OP_TYPE_SEND, IRDMA_UDA_QPSQ_OPCODE) |
- LS_64(1, IRDMA_UDA_QPSQ_SIGCOMPL) |
- LS_64(info->do_lpb, IRDMA_UDA_QPSQ_DOLOOPBACK) |
- LS_64(qp->qp_uk.swqe_polarity, IRDMA_UDA_QPSQ_VALID);
+ hdr[1] = FIELD_PREP(IRDMA_UDA_QPSQ_OPCODE, IRDMA_OP_TYPE_SEND) |
+ FIELD_PREP(IRDMA_UDA_QPSQ_SIGCOMPL, 1) |
+ FIELD_PREP(IRDMA_UDA_QPSQ_DOLOOPBACK, info->do_lpb) |
+ FIELD_PREP(IRDMA_UDA_QPSQ_VALID, qp->qp_uk.swqe_polarity);
/* Forth line of WQE descriptor */
set_64bit_val(wqe, IRDMA_BYTE_0, info->paddr);
set_64bit_val(wqe, IRDMA_BYTE_8,
- LS_64(info->len, IRDMAQPSQ_GEN1_FRAG_LEN));
+ FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, info->len));
}
set_64bit_val(wqe, IRDMA_BYTE_16, hdr[0]);
@@ -606,27 +607,27 @@
set_64bit_val(qp_ctx, IRDMA_BYTE_8, qp->sq_pa);
set_64bit_val(qp_ctx, IRDMA_BYTE_16, qp->rq_pa);
set_64bit_val(qp_ctx, IRDMA_BYTE_24,
- LS_64(qp->hw_rq_size, IRDMAQPC_RQSIZE) |
- LS_64(qp->hw_sq_size, IRDMAQPC_SQSIZE));
+ FIELD_PREP(IRDMAQPC_RQSIZE, qp->hw_rq_size) |
+ FIELD_PREP(IRDMAQPC_SQSIZE, qp->hw_sq_size));
set_64bit_val(qp_ctx, IRDMA_BYTE_48,
- LS_64(rsrc->buf_size, IRDMAQPC_SNDMSS));
+ FIELD_PREP(IRDMAQPC_SNDMSS, rsrc->buf_size));
set_64bit_val(qp_ctx, IRDMA_BYTE_56, 0);
if (qp->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
set_64bit_val(qp_ctx, IRDMA_BYTE_64, 1);
set_64bit_val(qp_ctx, IRDMA_BYTE_136,
- LS_64(rsrc->cq_id, IRDMAQPC_TXCQNUM) |
- LS_64(rsrc->cq_id, IRDMAQPC_RXCQNUM));
+ FIELD_PREP(IRDMAQPC_TXCQNUM, rsrc->cq_id) |
+ FIELD_PREP(IRDMAQPC_RXCQNUM, rsrc->cq_id));
set_64bit_val(qp_ctx, IRDMA_BYTE_144,
- LS_64(rsrc->stats_idx, IRDMAQPC_STAT_INDEX));
+ FIELD_PREP(IRDMAQPC_STAT_INDEX, rsrc->stats_idx));
set_64bit_val(qp_ctx, IRDMA_BYTE_160,
- LS_64(1, IRDMAQPC_PRIVEN) |
- LS_64(rsrc->stats_idx_valid, IRDMAQPC_USESTATSINSTANCE));
+ FIELD_PREP(IRDMAQPC_PRIVEN, 1) |
+ FIELD_PREP(IRDMAQPC_USESTATSINSTANCE, rsrc->stats_idx_valid));
set_64bit_val(qp_ctx, IRDMA_BYTE_168,
- LS_64((uintptr_t)qp, IRDMAQPC_QPCOMPCTX));
+ FIELD_PREP(IRDMAQPC_QPCOMPCTX, (uintptr_t)qp));
set_64bit_val(qp_ctx, IRDMA_BYTE_176,
- LS_64(qp->sq_tph_val, IRDMAQPC_SQTPHVAL) |
- LS_64(qp->rq_tph_val, IRDMAQPC_RQTPHVAL) |
- LS_64(qp->qs_handle, IRDMAQPC_QSHANDLE));
+ FIELD_PREP(IRDMAQPC_SQTPHVAL, qp->sq_tph_val) |
+ FIELD_PREP(IRDMAQPC_RQTPHVAL, qp->rq_tph_val) |
+ FIELD_PREP(IRDMAQPC_QSHANDLE, qp->qs_handle));
irdma_debug_buf(rsrc->dev, IRDMA_DEBUG_PUDA, "PUDA QP CONTEXT", qp_ctx,
IRDMA_QP_CTX_SIZE);
@@ -655,11 +656,11 @@
set_64bit_val(wqe, IRDMA_BYTE_40, qp->shadow_area_pa);
hdr = qp->qp_uk.qp_id |
- LS_64(IRDMA_CQP_OP_CREATE_QP, IRDMA_CQPSQ_OPCODE) |
- LS_64(IRDMA_QP_TYPE_UDA, IRDMA_CQPSQ_QP_QPTYPE) |
- LS_64(1, IRDMA_CQPSQ_QP_CQNUMVALID) |
- LS_64(2, IRDMA_CQPSQ_QP_NEXTIWSTATE) |
- LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_QP) |
+ FIELD_PREP(IRDMA_CQPSQ_QP_QPTYPE, IRDMA_QP_TYPE_UDA) |
+ FIELD_PREP(IRDMA_CQPSQ_QP_CQNUMVALID, 1) |
+ FIELD_PREP(IRDMA_CQPSQ_QP_NEXTIWSTATE, 2) |
+ FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
@@ -768,20 +769,19 @@
set_64bit_val(wqe, IRDMA_BYTE_0, cq->cq_uk.cq_size);
set_64bit_val(wqe, IRDMA_BYTE_8, RS_64_1(cq, 1));
set_64bit_val(wqe, IRDMA_BYTE_16,
- LS_64(cq->shadow_read_threshold,
- IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD));
+ FIELD_PREP(IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD, cq->shadow_read_threshold));
set_64bit_val(wqe, IRDMA_BYTE_32, cq->cq_pa);
set_64bit_val(wqe, IRDMA_BYTE_40, cq->shadow_area_pa);
set_64bit_val(wqe, IRDMA_BYTE_56,
- LS_64(cq->tph_val, IRDMA_CQPSQ_TPHVAL) |
- LS_64(cq->vsi->vsi_idx, IRDMA_CQPSQ_VSIIDX));
+ FIELD_PREP(IRDMA_CQPSQ_TPHVAL, cq->tph_val) |
+ FIELD_PREP(IRDMA_CQPSQ_VSIIDX, cq->vsi->vsi_idx));
hdr = cq->cq_uk.cq_id |
- LS_64(IRDMA_CQP_OP_CREATE_CQ, IRDMA_CQPSQ_OPCODE) |
- LS_64(1, IRDMA_CQPSQ_CQ_CHKOVERFLOW) |
- LS_64(1, IRDMA_CQPSQ_CQ_ENCEQEMASK) |
- LS_64(1, IRDMA_CQPSQ_CQ_CEQIDVALID) |
- LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_CQ) |
+ FIELD_PREP(IRDMA_CQPSQ_CQ_CHKOVERFLOW, 1) |
+ FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, 1) |
+ FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, 1) |
+ FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
@@ -962,13 +962,13 @@
case PUDA_QP_CREATED:
irdma_qp_rem_qos(&rsrc->qp);
- if (!(reset || dev->no_cqp))
+ if (!reset)
irdma_puda_free_qp(rsrc);
irdma_free_dma_mem(dev->hw, &rsrc->qpmem);
/* fallthrough */
case PUDA_CQ_CREATED:
- if (!(reset || dev->no_cqp))
+ if (!reset)
irdma_puda_free_cq(rsrc);
irdma_free_dma_mem(dev->hw, &rsrc->cqmem);
@@ -1007,7 +1007,7 @@
unsigned long flags;
buf_mem.size = count * sizeof(struct irdma_puda_buf);
- buf_mem.va = kzalloc(buf_mem.size, GFP_ATOMIC);
+ buf_mem.va = kzalloc(buf_mem.size, GFP_KERNEL);
if (!buf_mem.va) {
irdma_debug(rsrc->dev, IRDMA_DEBUG_PUDA,
"error virt_mem for buf\n");
@@ -1115,7 +1115,7 @@
return -EOPNOTSUPP;
}
vmem->size = pudasize + sqwridsize + rqwridsize;
- vmem->va = kzalloc(vmem->size, GFP_ATOMIC);
+ vmem->va = kzalloc(vmem->size, GFP_KERNEL);
if (!vmem->va)
return -ENOMEM;
@@ -1224,16 +1224,16 @@
if (qp->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
get_64bit_val(wqe, IRDMA_BYTE_8, &offset8);
if (offset24)
- offset8 &= ~LS_64(1, IRDMAQPSQ_VALID);
+ offset8 &= ~FIELD_PREP(IRDMAQPSQ_VALID, 1);
else
- offset8 |= LS_64(1, IRDMAQPSQ_VALID);
+ offset8 |= FIELD_PREP(IRDMAQPSQ_VALID, 1);
set_64bit_val(wqe, IRDMA_BYTE_8, offset8);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
}
if (offset24)
offset24 = 0;
else
- offset24 = LS_64(1, IRDMAQPSQ_VALID);
+ offset24 = FIELD_PREP(IRDMAQPSQ_VALID, 1);
set_64bit_val(wqe, IRDMA_BYTE_24, offset24);
}
diff --git a/sys/dev/irdma/irdma_type.h b/sys/dev/irdma/irdma_type.h
--- a/sys/dev/irdma/irdma_type.h
+++ b/sys/dev/irdma/irdma_type.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2021 Intel Corporation
+ * Copyright (c) 2015 - 2022 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -35,7 +35,9 @@
#ifndef IRDMA_TYPE_H
#define IRDMA_TYPE_H
+
#include "osdep.h"
+
#include "irdma.h"
#include "irdma_user.h"
#include "irdma_hmc.h"
@@ -132,11 +134,6 @@
MPA_REQ_RSP = 0x04,
};
-enum irdma_qp_event_type {
- IRDMA_QP_EVENT_CATASTROPHIC,
- IRDMA_QP_EVENT_ACCESS_ERR,
-};
-
enum irdma_hw_stats_index {
/* gen1 - 32-bit */
IRDMA_HW_STAT_INDEX_IP4RXDISCARD = 0,
@@ -176,22 +173,21 @@
IRDMA_HW_STAT_INDEX_RDMATXWRS = 33,
IRDMA_HW_STAT_INDEX_RDMAVBND = 34,
IRDMA_HW_STAT_INDEX_RDMAVINV = 35,
+ IRDMA_HW_STAT_INDEX_IP4RXMCOCTS = 36,
+ IRDMA_HW_STAT_INDEX_IP4TXMCOCTS = 37,
+ IRDMA_HW_STAT_INDEX_IP6RXMCOCTS = 38,
+ IRDMA_HW_STAT_INDEX_IP6TXMCOCTS = 39,
+ IRDMA_HW_STAT_INDEX_UDPRXPKTS = 40,
+ IRDMA_HW_STAT_INDEX_UDPTXPKTS = 41,
+ IRDMA_HW_STAT_INDEX_MAX_GEN_1 = 42, /* Must be same value as next entry */
- IRDMA_HW_STAT_INDEX_MAX_GEN_1 = 36, /* Must be same value as next entry */
-
- /* gen2 - 32-bit */
- IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED = 36,
- IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED = 37,
- IRDMA_HW_STAT_INDEX_TXNPCNPSENT = 38,
/* gen2 - 64-bit */
- IRDMA_HW_STAT_INDEX_IP4RXMCOCTS = 39,
- IRDMA_HW_STAT_INDEX_IP4TXMCOCTS = 40,
- IRDMA_HW_STAT_INDEX_IP6RXMCOCTS = 41,
- IRDMA_HW_STAT_INDEX_IP6TXMCOCTS = 42,
- IRDMA_HW_STAT_INDEX_UDPRXPKTS = 43,
- IRDMA_HW_STAT_INDEX_UDPTXPKTS = 44,
- IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS = 45,
+ IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS = 42,
+ /* gen2 - 32-bit */
+ IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED = 43,
+ IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED = 44,
+ IRDMA_HW_STAT_INDEX_TXNPCNPSENT = 45,
IRDMA_HW_STAT_INDEX_MAX_GEN_2 = 46,
};
@@ -331,8 +327,8 @@
struct irdma_stats_gather_info {
bool use_hmc_fcn_index:1;
bool use_stats_inst:1;
- u8 hmc_fcn_index;
- u8 stats_inst_index;
+ u16 hmc_fcn_index;
+ u16 stats_inst_index;
struct irdma_dma_mem stats_buff_mem;
void *gather_stats_va;
void *last_gather_stats_va;
@@ -524,14 +520,14 @@
struct irdma_stats_inst_info {
bool use_hmc_fcn_index;
- u8 hmc_fn_id;
- u8 stats_idx;
+ u16 hmc_fn_id;
+ u16 stats_idx;
};
struct irdma_up_info {
u8 map[8];
u8 cnp_up_override;
- u8 hmc_fcn_idx;
+ u16 hmc_fcn_idx;
bool use_vlan:1;
bool use_cnp_up_override:1;
};
@@ -585,7 +581,7 @@
u16 qs_handle;
};
-#define IRDMA_INVALID_FCN_ID 0xff
+#define IRDMA_INVALID_STATS_IDX 0xff
struct irdma_sc_vsi {
u16 vsi_idx;
struct irdma_sc_dev *dev;
@@ -598,11 +594,10 @@
struct irdma_puda_rsrc *ieq;
u32 exception_lan_q;
u16 mtu;
- u16 vm_id;
- u8 fcn_id;
enum irdma_vm_vf_type vm_vf_type;
- bool stats_fcn_id_alloc:1;
+ bool stats_inst_alloc:1;
bool tc_change_pending:1;
+ bool mtu_change_pending:1;
struct irdma_vsi_pestat *pestat;
ATOMIC qp_suspend_reqs;
int (*register_qset)(struct irdma_sc_vsi *vsi,
@@ -611,18 +606,17 @@
struct irdma_ws_node *tc_node);
struct irdma_config_check cfg_check[IRDMA_MAX_USER_PRIORITY];
bool tc_print_warning[IRDMA_MAX_TRAFFIC_CLASS];
- struct irdma_qos qos[IRDMA_MAX_USER_PRIORITY];
u8 qos_rel_bw;
u8 qos_prio_type;
+ u16 stats_idx;
u8 dscp_map[IRDMA_DSCP_NUM_VAL];
+ struct irdma_qos qos[IRDMA_MAX_USER_PRIORITY];
bool dscp_mode:1;
};
struct irdma_sc_dev {
struct list_head cqp_cmd_head; /* head of the CQP command list */
- bool volatile no_cqp;
spinlock_t cqp_lock; /* protect CQP list access */
- bool fcn_id_array[IRDMA_MAX_STATS_COUNT];
struct irdma_dma_mem vf_fpm_query_buf[IRDMA_MAX_PE_ENA_VF_COUNT];
u64 fpm_query_buf_pa;
u64 fpm_commit_buf_pa;
@@ -640,7 +634,7 @@
u32 IOMEM *hw_regs[IRDMA_MAX_REGS];
u32 ceq_itr; /* Interrupt throttle, usecs between interrupts: 0 disabled. 2 - 8160 */
u64 hw_masks[IRDMA_MAX_MASKS];
- u64 hw_shifts[IRDMA_MAX_SHIFTS];
+ u8 hw_shifts[IRDMA_MAX_SHIFTS];
const struct irdma_hw_stat_map *hw_stats_map;
u64 feature_info[IRDMA_MAX_FEATURES];
u64 cqp_cmd_stats[IRDMA_MAX_CQP_OPS];
@@ -656,7 +650,7 @@
struct mutex ws_mutex; /* ws tree mutex */
u32 debug_mask;
u16 num_vfs;
- u8 hmc_fn_id;
+ u16 hmc_fn_id;
u8 vf_id;
bool vchnl_up:1;
bool ceq_valid:1;
@@ -750,7 +744,6 @@
u16 exception_lan_q;
u16 pf_data_vsi_num;
enum irdma_vm_vf_type vm_vf_type;
- u16 vm_id;
int (*register_qset)(struct irdma_sc_vsi *vsi,
struct irdma_ws_node *tc_node);
void (*unregister_qset)(struct irdma_sc_vsi *vsi,
@@ -760,7 +753,7 @@
struct irdma_vsi_stats_info {
struct irdma_vsi_pestat *pestat;
u8 fcn_id;
- bool alloc_fcn_id;
+ bool alloc_stats_inst;
};
struct irdma_device_init_info {
@@ -771,7 +764,7 @@
struct irdma_hw *hw;
void IOMEM *bar0;
u16 max_vfs;
- u8 hmc_fn_id;
+ u16 hmc_fn_id;
u32 debug_mask;
};
@@ -852,10 +845,9 @@
struct irdma_roce_offload_info {
u16 p_key;
- u16 err_rq_idx;
+ u32 err_rq_idx;
u32 qkey;
u32 dest_qp;
- u32 local_qp;
u8 roce_tver;
u8 ack_credits;
u8 err_rq_idx_valid;
@@ -888,7 +880,7 @@
u8 ddp_ver;
u8 rdmap_ver;
u8 iwarp_mode;
- u16 err_rq_idx;
+ u32 err_rq_idx;
u32 pd_id;
u16 ord_size;
u16 ird_size;
@@ -976,7 +968,7 @@
u32 send_cq_num;
u32 rcv_cq_num;
u32 rem_endpoint_idx;
- u8 stats_idx;
+ u16 stats_idx;
bool srq_valid:1;
bool tcp_info_valid:1;
bool iwarp_info_valid:1;
@@ -987,8 +979,8 @@
struct irdma_aeqe_info {
u64 compl_ctx;
u32 qp_cq_id;
+ u32 wqe_idx;
u16 ae_id;
- u16 wqe_idx;
u8 tcp_state;
u8 iwarp_state;
bool qp:1;
@@ -1013,7 +1005,8 @@
bool remote_access:1;
bool use_hmc_fcn_index:1;
bool use_pf_rid:1;
- u8 hmc_fcn_index;
+ bool all_memory:1;
+ u16 hmc_fcn_index;
};
struct irdma_mw_alloc_info {
@@ -1038,8 +1031,9 @@
u32 pd_id;
irdma_stag_key stag_key;
bool use_hmc_fcn_index:1;
- u8 hmc_fcn_index;
+ u16 hmc_fcn_index;
bool use_pf_rid:1;
+ bool all_memory:1;
};
struct irdma_fast_reg_stag_info {
@@ -1061,7 +1055,7 @@
bool signaled:1;
bool push_wqe:1;
bool use_hmc_fcn_index:1;
- u8 hmc_fcn_index;
+ u16 hmc_fcn_index;
bool use_pf_rid:1;
bool defer_flag:1;
};
@@ -1231,7 +1225,7 @@
void irdma_check_cqp_progress(struct irdma_cqp_timeout *cqp_timeout,
struct irdma_sc_dev *dev);
int irdma_sc_cqp_create(struct irdma_sc_cqp *cqp, u16 *maj_err, u16 *min_err);
-int irdma_sc_cqp_destroy(struct irdma_sc_cqp *cqp);
+int irdma_sc_cqp_destroy(struct irdma_sc_cqp *cqp, bool free_hwcqp);
int irdma_sc_cqp_init(struct irdma_sc_cqp *cqp,
struct irdma_cqp_init_info *info);
void irdma_sc_cqp_post_sq(struct irdma_sc_cqp *cqp);
@@ -1249,10 +1243,10 @@
int irdma_sc_qp_modify(struct irdma_sc_qp *qp,
struct irdma_modify_qp_info *info, u64 scratch,
bool post_sq);
-int irdma_sc_send_lsmm(struct irdma_sc_qp *qp, void *lsmm_buf, u32 size,
- irdma_stag);
-int irdma_sc_send_lsmm_nostag(struct irdma_sc_qp *qp, void *lsmm_buf, u32 size);
-int irdma_sc_send_rtt(struct irdma_sc_qp *qp, bool read);
+void irdma_sc_send_lsmm(struct irdma_sc_qp *qp, void *lsmm_buf, u32 size,
+ irdma_stag stag);
+void irdma_sc_send_lsmm_nostag(struct irdma_sc_qp *qp, void *lsmm_buf, u32 size);
+void irdma_sc_send_rtt(struct irdma_sc_qp *qp, bool read);
void irdma_sc_qp_setctx(struct irdma_sc_qp *qp, __le64 *qp_ctx,
struct irdma_qp_host_ctx_info *info);
void irdma_sc_qp_setctx_roce(struct irdma_sc_qp *qp, __le64 *qp_ctx,
@@ -1260,8 +1254,9 @@
int irdma_sc_cq_destroy(struct irdma_sc_cq *cq, u64 scratch, bool post_sq);
int irdma_sc_cq_init(struct irdma_sc_cq *cq, struct irdma_cq_init_info *info);
void irdma_sc_cq_resize(struct irdma_sc_cq *cq, struct irdma_modify_cq_info *info);
+int irdma_sc_aeq_destroy(struct irdma_sc_aeq *aeq, u64 scratch, bool post_sq);
int irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp *cqp, u64 scratch,
- u8 hmc_fn_id, bool post_sq,
+ u16 hmc_fn_id, bool post_sq,
bool poll_registers);
void sc_vsi_update_stats(struct irdma_sc_vsi *vsi);
@@ -1411,7 +1406,7 @@
struct irdma_sc_cqp *cqp;
void *fpm_val_va;
u64 fpm_val_pa;
- u8 hmc_fn_id;
+ u16 hmc_fn_id;
u64 scratch;
} query_fpm_val;
@@ -1419,7 +1414,7 @@
struct irdma_sc_cqp *cqp;
void *fpm_val_va;
u64 fpm_val_pa;
- u8 hmc_fn_id;
+ u16 hmc_fn_id;
u64 scratch;
} commit_fpm_val;
diff --git a/sys/dev/irdma/irdma_uda.c b/sys/dev/irdma/irdma_uda.c
--- a/sys/dev/irdma/irdma_uda.c
+++ b/sys/dev/irdma/irdma_uda.c
@@ -66,35 +66,35 @@
LS_64_1(info->mac_addr[1], 48) |
LS_64_1(info->mac_addr[0], 56));
- qw1 = LS_64(info->pd_idx, IRDMA_UDA_CQPSQ_MAV_PDINDEXLO) |
- LS_64(info->tc_tos, IRDMA_UDA_CQPSQ_MAV_TC) |
- LS_64(info->vlan_tag, IRDMA_UDAQPC_VLANTAG);
+ qw1 = FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_PDINDEXLO, info->pd_idx) |
+ FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_TC, info->tc_tos) |
+ FIELD_PREP(IRDMA_UDAQPC_VLANTAG, info->vlan_tag);
- qw2 = LS_64(info->dst_arpindex, IRDMA_UDA_CQPSQ_MAV_ARPINDEX) |
- LS_64(info->flow_label, IRDMA_UDA_CQPSQ_MAV_FLOWLABEL) |
- LS_64(info->hop_ttl, IRDMA_UDA_CQPSQ_MAV_HOPLIMIT) |
- LS_64(info->pd_idx >> 16, IRDMA_UDA_CQPSQ_MAV_PDINDEXHI);
+ qw2 = FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ARPINDEX, info->dst_arpindex) |
+ FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_FLOWLABEL, info->flow_label) |
+ FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_HOPLIMIT, info->hop_ttl) |
+ FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_PDINDEXHI, info->pd_idx >> 16);
if (!info->ipv4_valid) {
set_64bit_val(wqe, IRDMA_BYTE_40,
- LS_64(info->dest_ip_addr[0], IRDMA_UDA_CQPSQ_MAV_ADDR0) |
- LS_64(info->dest_ip_addr[1], IRDMA_UDA_CQPSQ_MAV_ADDR1));
+ FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR0, info->dest_ip_addr[0]) |
+ FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR1, info->dest_ip_addr[1]));
set_64bit_val(wqe, IRDMA_BYTE_32,
- LS_64(info->dest_ip_addr[2], IRDMA_UDA_CQPSQ_MAV_ADDR2) |
- LS_64(info->dest_ip_addr[3], IRDMA_UDA_CQPSQ_MAV_ADDR3));
+ FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR2, info->dest_ip_addr[2]) |
+ FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR3, info->dest_ip_addr[3]));
set_64bit_val(wqe, IRDMA_BYTE_56,
- LS_64(info->src_ip_addr[0], IRDMA_UDA_CQPSQ_MAV_ADDR0) |
- LS_64(info->src_ip_addr[1], IRDMA_UDA_CQPSQ_MAV_ADDR1));
+ FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR0, info->src_ip_addr[0]) |
+ FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR1, info->src_ip_addr[1]));
set_64bit_val(wqe, IRDMA_BYTE_48,
- LS_64(info->src_ip_addr[2], IRDMA_UDA_CQPSQ_MAV_ADDR2) |
- LS_64(info->src_ip_addr[3], IRDMA_UDA_CQPSQ_MAV_ADDR3));
+ FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR2, info->src_ip_addr[2]) |
+ FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR3, info->src_ip_addr[3]));
} else {
set_64bit_val(wqe, IRDMA_BYTE_32,
- LS_64(info->dest_ip_addr[0], IRDMA_UDA_CQPSQ_MAV_ADDR3));
+ FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR3, info->dest_ip_addr[0]));
set_64bit_val(wqe, IRDMA_BYTE_48,
- LS_64(info->src_ip_addr[0], IRDMA_UDA_CQPSQ_MAV_ADDR3));
+ FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR3, info->src_ip_addr[0]));
}
set_64bit_val(wqe, IRDMA_BYTE_8, qw1);
@@ -104,13 +104,12 @@
set_64bit_val(
wqe, IRDMA_BYTE_24,
- LS_64(cqp->polarity, IRDMA_UDA_CQPSQ_MAV_WQEVALID) |
- LS_64(op, IRDMA_UDA_CQPSQ_MAV_OPCODE) |
- LS_64(info->do_lpbk, IRDMA_UDA_CQPSQ_MAV_DOLOOPBACKK) |
- LS_64(info->ipv4_valid, IRDMA_UDA_CQPSQ_MAV_IPV4VALID) |
- LS_64(info->ah_idx, IRDMA_UDA_CQPSQ_MAV_AVIDX) |
- LS_64(info->insert_vlan_tag,
- IRDMA_UDA_CQPSQ_MAV_INSERTVLANTAG));
+ FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_WQEVALID, cqp->polarity) |
+ FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_OPCODE, op) |
+ FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_DOLOOPBACKK, info->do_lpbk) |
+ FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_IPV4VALID, info->ipv4_valid) |
+ FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_AVIDX, info->ah_idx) |
+ FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_INSERTVLANTAG, info->insert_vlan_tag));
irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "MANAGE_AH WQE", wqe,
IRDMA_CQP_WQE_SIZE * 8);
@@ -137,9 +136,9 @@
if (entry_info->valid_entry) {
set_64bit_val((__le64 *) info->dma_mem_mc.va,
ctx_idx * sizeof(u64),
- LS_64(entry_info->dest_port, IRDMA_UDA_MGCTX_DESTPORT) |
- LS_64(entry_info->valid_entry, IRDMA_UDA_MGCTX_VALIDENT) |
- LS_64(entry_info->qp_id, IRDMA_UDA_MGCTX_QPID));
+ FIELD_PREP(IRDMA_UDA_MGCTX_DESTPORT, entry_info->dest_port) |
+ FIELD_PREP(IRDMA_UDA_MGCTX_VALIDENT, entry_info->valid_entry) |
+ FIELD_PREP(IRDMA_UDA_MGCTX_QPID, entry_info->qp_id));
ctx_idx++;
}
}
@@ -179,8 +178,8 @@
set_64bit_val(wqe, IRDMA_BYTE_32, info->dma_mem_mc.pa);
set_64bit_val(wqe, IRDMA_BYTE_16,
- LS_64(info->vlan_id, IRDMA_UDA_CQPSQ_MG_VLANID) |
- LS_64(info->qs_handle, IRDMA_UDA_CQPSQ_QS_HANDLE));
+ FIELD_PREP(IRDMA_UDA_CQPSQ_MG_VLANID, info->vlan_id) |
+ FIELD_PREP(IRDMA_UDA_CQPSQ_QS_HANDLE, info->qs_handle));
set_64bit_val(wqe, IRDMA_BYTE_0, LS_64_1(info->dest_mac_addr[5], 0) |
LS_64_1(info->dest_mac_addr[4], 8) |
LS_64_1(info->dest_mac_addr[3], 16) |
@@ -188,28 +187,28 @@
LS_64_1(info->dest_mac_addr[1], 32) |
LS_64_1(info->dest_mac_addr[0], 40));
set_64bit_val(wqe, IRDMA_BYTE_8,
- LS_64(info->hmc_fcn_id, IRDMA_UDA_CQPSQ_MG_HMC_FCN_ID));
+ FIELD_PREP(IRDMA_UDA_CQPSQ_MG_HMC_FCN_ID, info->hmc_fcn_id));
if (!info->ipv4_valid) {
set_64bit_val(wqe, IRDMA_BYTE_56,
- LS_64(info->dest_ip_addr[0], IRDMA_UDA_CQPSQ_MAV_ADDR0) |
- LS_64(info->dest_ip_addr[1], IRDMA_UDA_CQPSQ_MAV_ADDR1));
+ FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR0, info->dest_ip_addr[0]) |
+ FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR1, info->dest_ip_addr[1]));
set_64bit_val(wqe, IRDMA_BYTE_48,
- LS_64(info->dest_ip_addr[2], IRDMA_UDA_CQPSQ_MAV_ADDR2) |
- LS_64(info->dest_ip_addr[3], IRDMA_UDA_CQPSQ_MAV_ADDR3));
+ FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR2, info->dest_ip_addr[2]) |
+ FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR3, info->dest_ip_addr[3]));
} else {
set_64bit_val(wqe, IRDMA_BYTE_48,
- LS_64(info->dest_ip_addr[0], IRDMA_UDA_CQPSQ_MAV_ADDR3));
+ FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR3, info->dest_ip_addr[0]));
}
irdma_wmb(); /* need write memory block before writing the WQE header. */
set_64bit_val(wqe, IRDMA_BYTE_24,
- LS_64(cqp->polarity, IRDMA_UDA_CQPSQ_MG_WQEVALID) |
- LS_64(op, IRDMA_UDA_CQPSQ_MG_OPCODE) |
- LS_64(info->mg_id, IRDMA_UDA_CQPSQ_MG_MGIDX) |
- LS_64(info->vlan_valid, IRDMA_UDA_CQPSQ_MG_VLANVALID) |
- LS_64(info->ipv4_valid, IRDMA_UDA_CQPSQ_MG_IPV4VALID));
+ FIELD_PREP(IRDMA_UDA_CQPSQ_MG_WQEVALID, cqp->polarity) |
+ FIELD_PREP(IRDMA_UDA_CQPSQ_MG_OPCODE, op) |
+ FIELD_PREP(IRDMA_UDA_CQPSQ_MG_MGIDX, info->mg_id) |
+ FIELD_PREP(IRDMA_UDA_CQPSQ_MG_VLANVALID, info->vlan_valid) |
+ FIELD_PREP(IRDMA_UDA_CQPSQ_MG_IPV4VALID, info->ipv4_valid));
irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "MANAGE_MCG WQE", wqe,
IRDMA_CQP_WQE_SIZE * 8);
diff --git a/sys/dev/irdma/irdma_uda_d.h b/sys/dev/irdma/irdma_uda_d.h
--- a/sys/dev/irdma/irdma_uda_d.h
+++ b/sys/dev/irdma/irdma_uda_d.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2019 Intel Corporation
+ * Copyright (c) 2016 - 2021 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -35,381 +35,220 @@
#ifndef IRDMA_UDA_D_H
#define IRDMA_UDA_D_H
-
/* L4 packet type */
#define IRDMA_E_UDA_SQ_L4T_UNKNOWN 0
#define IRDMA_E_UDA_SQ_L4T_TCP 1
#define IRDMA_E_UDA_SQ_L4T_SCTP 2
#define IRDMA_E_UDA_SQ_L4T_UDP 3
-
/* Inner IP header type */
#define IRDMA_E_UDA_SQ_IIPT_UNKNOWN 0
#define IRDMA_E_UDA_SQ_IIPT_IPV6 1
#define IRDMA_E_UDA_SQ_IIPT_IPV4_NO_CSUM 2
#define IRDMA_E_UDA_SQ_IIPT_IPV4_CSUM 3
-
-/* UDA defined fields for transmit descriptors */
#define IRDMA_UDA_QPSQ_PUSHWQE_S 56
-#define IRDMA_UDA_QPSQ_PUSHWQE_M BIT_ULL(IRDMA_UDA_QPSQ_PUSHWQE_S)
-
+#define IRDMA_UDA_QPSQ_PUSHWQE BIT_ULL(56)
#define IRDMA_UDA_QPSQ_INLINEDATAFLAG_S 57
-#define IRDMA_UDA_QPSQ_INLINEDATAFLAG_M \
- BIT_ULL(IRDMA_UDA_QPSQ_INLINEDATAFLAG_S)
-
+#define IRDMA_UDA_QPSQ_INLINEDATAFLAG BIT_ULL(57)
#define IRDMA_UDA_QPSQ_INLINEDATALEN_S 48
-#define IRDMA_UDA_QPSQ_INLINEDATALEN_M \
- ((u64)0xff << IRDMA_UDA_QPSQ_INLINEDATALEN_S)
-
+#define IRDMA_UDA_QPSQ_INLINEDATALEN GENMASK_ULL(55, 48)
#define IRDMA_UDA_QPSQ_ADDFRAGCNT_S 38
-#define IRDMA_UDA_QPSQ_ADDFRAGCNT_M \
- ((u64)0x0F << IRDMA_UDA_QPSQ_ADDFRAGCNT_S)
-
+#define IRDMA_UDA_QPSQ_ADDFRAGCNT GENMASK_ULL(41, 38)
#define IRDMA_UDA_QPSQ_IPFRAGFLAGS_S 42
-#define IRDMA_UDA_QPSQ_IPFRAGFLAGS_M \
- ((u64)0x3 << IRDMA_UDA_QPSQ_IPFRAGFLAGS_S)
-
+#define IRDMA_UDA_QPSQ_IPFRAGFLAGS GENMASK_ULL(43, 42)
#define IRDMA_UDA_QPSQ_NOCHECKSUM_S 45
-#define IRDMA_UDA_QPSQ_NOCHECKSUM_M \
- BIT_ULL(IRDMA_UDA_QPSQ_NOCHECKSUM_S)
-
+#define IRDMA_UDA_QPSQ_NOCHECKSUM BIT_ULL(45)
#define IRDMA_UDA_QPSQ_AHIDXVALID_S 46
-#define IRDMA_UDA_QPSQ_AHIDXVALID_M \
- BIT_ULL(IRDMA_UDA_QPSQ_AHIDXVALID_S)
-
+#define IRDMA_UDA_QPSQ_AHIDXVALID BIT_ULL(46)
#define IRDMA_UDA_QPSQ_LOCAL_FENCE_S 61
-#define IRDMA_UDA_QPSQ_LOCAL_FENCE_M \
- BIT_ULL(IRDMA_UDA_QPSQ_LOCAL_FENCE_S)
-
+#define IRDMA_UDA_QPSQ_LOCAL_FENCE BIT_ULL(61)
#define IRDMA_UDA_QPSQ_AHIDX_S 0
-#define IRDMA_UDA_QPSQ_AHIDX_M ((u64)0x1ffff << IRDMA_UDA_QPSQ_AHIDX_S)
-
+#define IRDMA_UDA_QPSQ_AHIDX GENMASK_ULL(16, 0)
#define IRDMA_UDA_QPSQ_PROTOCOL_S 16
-#define IRDMA_UDA_QPSQ_PROTOCOL_M \
- ((u64)0xff << IRDMA_UDA_QPSQ_PROTOCOL_S)
-
+#define IRDMA_UDA_QPSQ_PROTOCOL GENMASK_ULL(23, 16)
#define IRDMA_UDA_QPSQ_EXTHDRLEN_S 32
-#define IRDMA_UDA_QPSQ_EXTHDRLEN_M \
- ((u64)0x1ff << IRDMA_UDA_QPSQ_EXTHDRLEN_S)
-
+#define IRDMA_UDA_QPSQ_EXTHDRLEN GENMASK_ULL(40, 32)
#define IRDMA_UDA_QPSQ_MULTICAST_S 63
-#define IRDMA_UDA_QPSQ_MULTICAST_M \
- BIT_ULL(IRDMA_UDA_QPSQ_MULTICAST_S)
-
+#define IRDMA_UDA_QPSQ_MULTICAST BIT_ULL(63)
#define IRDMA_UDA_QPSQ_MACLEN_S 56
-#define IRDMA_UDA_QPSQ_MACLEN_M \
- ((u64)0x7f << IRDMA_UDA_QPSQ_MACLEN_S)
+#define IRDMA_UDA_QPSQ_MACLEN GENMASK_ULL(62, 56)
#define IRDMA_UDA_QPSQ_MACLEN_LINE 2
-
#define IRDMA_UDA_QPSQ_IPLEN_S 48
-#define IRDMA_UDA_QPSQ_IPLEN_M \
- ((u64)0x7f << IRDMA_UDA_QPSQ_IPLEN_S)
+#define IRDMA_UDA_QPSQ_IPLEN GENMASK_ULL(54, 48)
#define IRDMA_UDA_QPSQ_IPLEN_LINE 2
-
#define IRDMA_UDA_QPSQ_L4T_S 30
-#define IRDMA_UDA_QPSQ_L4T_M ((u64)0x3 << IRDMA_UDA_QPSQ_L4T_S)
+#define IRDMA_UDA_QPSQ_L4T GENMASK_ULL(31, 30)
#define IRDMA_UDA_QPSQ_L4T_LINE 2
-
#define IRDMA_UDA_QPSQ_IIPT_S 28
-#define IRDMA_UDA_QPSQ_IIPT_M ((u64)0x3 << IRDMA_UDA_QPSQ_IIPT_S)
+#define IRDMA_UDA_QPSQ_IIPT GENMASK_ULL(29, 28)
#define IRDMA_UDA_QPSQ_IIPT_LINE 2
-
#define IRDMA_UDA_QPSQ_DO_LPB_LINE 3
-
#define IRDMA_UDA_QPSQ_FWD_PROG_CONFIRM_S 45
-#define IRDMA_UDA_QPSQ_FWD_PROG_CONFIRM_M \
- BIT_ULL(IRDMA_UDA_QPSQ_FWD_PROG_CONFIRM_S)
+#define IRDMA_UDA_QPSQ_FWD_PROG_CONFIRM BIT_ULL(45)
#define IRDMA_UDA_QPSQ_FWD_PROG_CONFIRM_LINE 3
-
#define IRDMA_UDA_QPSQ_IMMDATA_S 0
-#define IRDMA_UDA_QPSQ_IMMDATA_M \
- ((u64)0xffffffffffffffff << IRDMA_UDA_QPSQ_IMMDATA_S)
-
+#define IRDMA_UDA_QPSQ_IMMDATA GENMASK_ULL(63, 0)
/* Byte Offset 0 */
#define IRDMA_UDAQPC_IPV4_S 3
-#define IRDMA_UDAQPC_IPV4_M BIT_ULL(IRDMAQPC_IPV4_S)
-
+#define IRDMA_UDAQPC_IPV4 BIT_ULL(3)
#define IRDMA_UDAQPC_INSERTVLANTAG_S 5
-#define IRDMA_UDAQPC_INSERTVLANTAG_M BIT_ULL(IRDMA_UDAQPC_INSERTVLANTAG_S)
-
+#define IRDMA_UDAQPC_INSERTVLANTAG BIT_ULL(5)
#define IRDMA_UDAQPC_ISQP1_S 6
-#define IRDMA_UDAQPC_ISQP1_M BIT_ULL(IRDMA_UDAQPC_ISQP1_S)
-
+#define IRDMA_UDAQPC_ISQP1 BIT_ULL(6)
#define IRDMA_UDAQPC_RQWQESIZE_S IRDMAQPC_RQWQESIZE_S
-#define IRDMA_UDAQPC_RQWQESIZE_M IRDMAQPC_RQWQESIZE_M
-
+#define IRDMA_UDAQPC_RQWQESIZE IRDMAQPC_RQWQESIZE
#define IRDMA_UDAQPC_ECNENABLE_S 14
-#define IRDMA_UDAQPC_ECNENABLE_M BIT_ULL(IRDMA_UDAQPC_ECNENABLE_S)
-
+#define IRDMA_UDAQPC_ECNENABLE BIT_ULL(14)
#define IRDMA_UDAQPC_PDINDEXHI_S 20
-#define IRDMA_UDAQPC_PDINDEXHI_M ((u64)3 << IRDMA_UDAQPC_PDINDEXHI_S)
-
+#define IRDMA_UDAQPC_PDINDEXHI GENMASK_ULL(21, 20)
#define IRDMA_UDAQPC_DCTCPENABLE_S 25
-#define IRDMA_UDAQPC_DCTCPENABLE_M BIT_ULL(IRDMA_UDAQPC_DCTCPENABLE_S)
-
+#define IRDMA_UDAQPC_DCTCPENABLE BIT_ULL(25)
#define IRDMA_UDAQPC_RCVTPHEN_S IRDMAQPC_RCVTPHEN_S
-#define IRDMA_UDAQPC_RCVTPHEN_M IRDMAQPC_RCVTPHEN_M
-
+#define IRDMA_UDAQPC_RCVTPHEN IRDMAQPC_RCVTPHEN
#define IRDMA_UDAQPC_XMITTPHEN_S IRDMAQPC_XMITTPHEN_S
-#define IRDMA_UDAQPC_XMITTPHEN_M IRDMAQPC_XMITTPHEN_M
-
+#define IRDMA_UDAQPC_XMITTPHEN IRDMAQPC_XMITTPHEN
#define IRDMA_UDAQPC_RQTPHEN_S IRDMAQPC_RQTPHEN_S
-#define IRDMA_UDAQPC_RQTPHEN_M IRDMAQPC_RQTPHEN_M
-
+#define IRDMA_UDAQPC_RQTPHEN IRDMAQPC_RQTPHEN
#define IRDMA_UDAQPC_SQTPHEN_S IRDMAQPC_SQTPHEN_S
-#define IRDMA_UDAQPC_SQTPHEN_M IRDMAQPC_SQTPHEN_M
-
+#define IRDMA_UDAQPC_SQTPHEN IRDMAQPC_SQTPHEN
#define IRDMA_UDAQPC_PPIDX_S IRDMAQPC_PPIDX_S
-#define IRDMA_UDAQPC_PPIDX_M IRDMAQPC_PPIDX_M
-
+#define IRDMA_UDAQPC_PPIDX IRDMAQPC_PPIDX
#define IRDMA_UDAQPC_PMENA_S IRDMAQPC_PMENA_S
-#define IRDMA_UDAQPC_PMENA_M IRDMAQPC_PMENA_M
-
+#define IRDMA_UDAQPC_PMENA IRDMAQPC_PMENA
#define IRDMA_UDAQPC_INSERTTAG2_S 11
-#define IRDMA_UDAQPC_INSERTTAG2_M BIT_ULL(IRDMA_UDAQPC_INSERTTAG2_S)
-
+#define IRDMA_UDAQPC_INSERTTAG2 BIT_ULL(11)
#define IRDMA_UDAQPC_INSERTTAG3_S 14
-#define IRDMA_UDAQPC_INSERTTAG3_M BIT_ULL(IRDMA_UDAQPC_INSERTTAG3_S)
-
+#define IRDMA_UDAQPC_INSERTTAG3 BIT_ULL(14)
#define IRDMA_UDAQPC_RQSIZE_S IRDMAQPC_RQSIZE_S
-#define IRDMA_UDAQPC_RQSIZE_M IRDMAQPC_RQSIZE_M
-
+#define IRDMA_UDAQPC_RQSIZE IRDMAQPC_RQSIZE
#define IRDMA_UDAQPC_SQSIZE_S IRDMAQPC_SQSIZE_S
-#define IRDMA_UDAQPC_SQSIZE_M IRDMAQPC_SQSIZE_M
-
+#define IRDMA_UDAQPC_SQSIZE IRDMAQPC_SQSIZE
#define IRDMA_UDAQPC_TXCQNUM_S IRDMAQPC_TXCQNUM_S
-#define IRDMA_UDAQPC_TXCQNUM_M IRDMAQPC_TXCQNUM_M
-
+#define IRDMA_UDAQPC_TXCQNUM IRDMAQPC_TXCQNUM
#define IRDMA_UDAQPC_RXCQNUM_S IRDMAQPC_RXCQNUM_S
-#define IRDMA_UDAQPC_RXCQNUM_M IRDMAQPC_RXCQNUM_M
-
+#define IRDMA_UDAQPC_RXCQNUM IRDMAQPC_RXCQNUM
#define IRDMA_UDAQPC_QPCOMPCTX_S IRDMAQPC_QPCOMPCTX_S
-#define IRDMA_UDAQPC_QPCOMPCTX_M IRDMAQPC_QPCOMPCTX_M
-
+#define IRDMA_UDAQPC_QPCOMPCTX IRDMAQPC_QPCOMPCTX
#define IRDMA_UDAQPC_SQTPHVAL_S IRDMAQPC_SQTPHVAL_S
-#define IRDMA_UDAQPC_SQTPHVAL_M IRDMAQPC_SQTPHVAL_M
-
+#define IRDMA_UDAQPC_SQTPHVAL IRDMAQPC_SQTPHVAL
#define IRDMA_UDAQPC_RQTPHVAL_S IRDMAQPC_RQTPHVAL_S
-#define IRDMA_UDAQPC_RQTPHVAL_M IRDMAQPC_RQTPHVAL_M
-
+#define IRDMA_UDAQPC_RQTPHVAL IRDMAQPC_RQTPHVAL
#define IRDMA_UDAQPC_QSHANDLE_S IRDMAQPC_QSHANDLE_S
-#define IRDMA_UDAQPC_QSHANDLE_M IRDMAQPC_QSHANDLE_M
-
+#define IRDMA_UDAQPC_QSHANDLE IRDMAQPC_QSHANDLE
#define IRDMA_UDAQPC_RQHDRRINGBUFSIZE_S 48
-#define IRDMA_UDAQPC_RQHDRRINGBUFSIZE_M \
- ((u64)0x3 << IRDMA_UDAQPC_RQHDRRINGBUFSIZE_S)
-
+#define IRDMA_UDAQPC_RQHDRRINGBUFSIZE GENMASK_ULL(49, 48)
#define IRDMA_UDAQPC_SQHDRRINGBUFSIZE_S 32
-#define IRDMA_UDAQPC_SQHDRRINGBUFSIZE_M \
- ((u64)0x3 << IRDMA_UDAQPC_SQHDRRINGBUFSIZE_S)
-
+#define IRDMA_UDAQPC_SQHDRRINGBUFSIZE GENMASK_ULL(33, 32)
#define IRDMA_UDAQPC_PRIVILEGEENABLE_S 25
-#define IRDMA_UDAQPC_PRIVILEGEENABLE_M \
- BIT_ULL(IRDMA_UDAQPC_PRIVILEGEENABLE_S)
-
+#define IRDMA_UDAQPC_PRIVILEGEENABLE BIT_ULL(25)
#define IRDMA_UDAQPC_USE_STATISTICS_INSTANCE_S 26
-#define IRDMA_UDAQPC_USE_STATISTICS_INSTANCE_M \
- BIT_ULL(IRDMA_UDAQPC_USE_STATISTICS_INSTANCE_S)
-
+#define IRDMA_UDAQPC_USE_STATISTICS_INSTANCE BIT_ULL(26)
#define IRDMA_UDAQPC_STATISTICS_INSTANCE_INDEX_S 0
-#define IRDMA_UDAQPC_STATISTICS_INSTANCE_INDEX_M \
- ((u64)0x7F << IRDMA_UDAQPC_STATISTICS_INSTANCE_INDEX_S)
-
+#define IRDMA_UDAQPC_STATISTICS_INSTANCE_INDEX GENMASK_ULL(6, 0)
#define IRDMA_UDAQPC_PRIVHDRGENENABLE_S 0
-#define IRDMA_UDAQPC_PRIVHDRGENENABLE_M \
- BIT_ULL(IRDMA_UDAQPC_PRIVHDRGENENABLE_S)
-
+#define IRDMA_UDAQPC_PRIVHDRGENENABLE BIT_ULL(0)
#define IRDMA_UDAQPC_RQHDRSPLITENABLE_S 3
-#define IRDMA_UDAQPC_RQHDRSPLITENABLE_M \
- BIT_ULL(IRDMA_UDAQPC_RQHDRSPLITENABLE_S)
-
+#define IRDMA_UDAQPC_RQHDRSPLITENABLE BIT_ULL(3)
#define IRDMA_UDAQPC_RQHDRRINGBUFENABLE_S 2
-#define IRDMA_UDAQPC_RQHDRRINGBUFENABLE_M \
- BIT_ULL(IRDMA_UDAQPC_RQHDRRINGBUFENABLE_S)
-
+#define IRDMA_UDAQPC_RQHDRRINGBUFENABLE BIT_ULL(2)
#define IRDMA_UDAQPC_SQHDRRINGBUFENABLE_S 1
-#define IRDMA_UDAQPC_SQHDRRINGBUFENABLE_M \
- BIT_ULL(IRDMA_UDAQPC_SQHDRRINGBUFENABLE_S)
-
+#define IRDMA_UDAQPC_SQHDRRINGBUFENABLE BIT_ULL(1)
#define IRDMA_UDAQPC_IPID_S 32
-#define IRDMA_UDAQPC_IPID_M ((u64)0xffff << IRDMA_UDAQPC_IPID_S)
-
+#define IRDMA_UDAQPC_IPID GENMASK_ULL(47, 32)
#define IRDMA_UDAQPC_SNDMSS_S 16
-#define IRDMA_UDAQPC_SNDMSS_M ((u64)0x3fff << IRDMA_UDAQPC_SNDMSS_S)
-
+#define IRDMA_UDAQPC_SNDMSS GENMASK_ULL(29, 16)
#define IRDMA_UDAQPC_VLANTAG_S 0
-#define IRDMA_UDAQPC_VLANTAG_M ((u64)0xffff << IRDMA_UDAQPC_VLANTAG_S)
-
-/* Address Handle */
+#define IRDMA_UDAQPC_VLANTAG GENMASK_ULL(15, 0)
#define IRDMA_UDA_CQPSQ_MAV_PDINDEXHI_S 20
-#define IRDMA_UDA_CQPSQ_MAV_PDINDEXHI_M \
- ((u64)0x3 << IRDMA_UDA_CQPSQ_MAV_PDINDEXHI_S)
-
+#define IRDMA_UDA_CQPSQ_MAV_PDINDEXHI GENMASK_ULL(21, 20)
#define IRDMA_UDA_CQPSQ_MAV_PDINDEXLO_S 48
-#define IRDMA_UDA_CQPSQ_MAV_PDINDEXLO_M \
- ((u64)0xffff << IRDMA_UDA_CQPSQ_MAV_PDINDEXLO_S)
-
+#define IRDMA_UDA_CQPSQ_MAV_PDINDEXLO GENMASK_ULL(63, 48)
#define IRDMA_UDA_CQPSQ_MAV_SRCMACADDRINDEX_S 24
-#define IRDMA_UDA_CQPSQ_MAV_SRCMACADDRINDEX_M \
- ((u64)0x3f << IRDMA_UDA_CQPSQ_MAV_SRCMACADDRINDEX_S)
-
+#define IRDMA_UDA_CQPSQ_MAV_SRCMACADDRINDEX GENMASK_ULL(29, 24)
#define IRDMA_UDA_CQPSQ_MAV_ARPINDEX_S 48
-#define IRDMA_UDA_CQPSQ_MAV_ARPINDEX_M \
- ((u64)0xffff << IRDMA_UDA_CQPSQ_MAV_ARPINDEX_S)
-
+#define IRDMA_UDA_CQPSQ_MAV_ARPINDEX GENMASK_ULL(63, 48)
#define IRDMA_UDA_CQPSQ_MAV_TC_S 32
-#define IRDMA_UDA_CQPSQ_MAV_TC_M ((u64)0xff << IRDMA_UDA_CQPSQ_MAV_TC_S)
-
+#define IRDMA_UDA_CQPSQ_MAV_TC GENMASK_ULL(39, 32)
#define IRDMA_UDA_CQPSQ_MAV_HOPLIMIT_S 32
-#define IRDMA_UDA_CQPSQ_MAV_HOPLIMIT_M \
- ((u64)0xff << IRDMA_UDA_CQPSQ_MAV_HOPLIMIT_S)
-
+#define IRDMA_UDA_CQPSQ_MAV_HOPLIMIT GENMASK_ULL(39, 32)
#define IRDMA_UDA_CQPSQ_MAV_FLOWLABEL_S 0
-#define IRDMA_UDA_CQPSQ_MAV_FLOWLABEL_M \
- ((u64)0xfffff << IRDMA_UDA_CQPSQ_MAV_FLOWLABEL_S)
-
+#define IRDMA_UDA_CQPSQ_MAV_FLOWLABEL GENMASK_ULL(19, 0)
#define IRDMA_UDA_CQPSQ_MAV_ADDR0_S 32
-#define IRDMA_UDA_CQPSQ_MAV_ADDR0_M \
- ((u64)0xffffffff << IRDMA_UDA_CQPSQ_MAV_ADDR0_S)
-
+#define IRDMA_UDA_CQPSQ_MAV_ADDR0 GENMASK_ULL(63, 32)
#define IRDMA_UDA_CQPSQ_MAV_ADDR1_S 0
-#define IRDMA_UDA_CQPSQ_MAV_ADDR1_M \
- ((u64)0xffffffff << IRDMA_UDA_CQPSQ_MAV_ADDR1_S)
-
+#define IRDMA_UDA_CQPSQ_MAV_ADDR1 GENMASK_ULL(31, 0)
#define IRDMA_UDA_CQPSQ_MAV_ADDR2_S 32
-#define IRDMA_UDA_CQPSQ_MAV_ADDR2_M \
- ((u64)0xffffffff << IRDMA_UDA_CQPSQ_MAV_ADDR2_S)
-
+#define IRDMA_UDA_CQPSQ_MAV_ADDR2 GENMASK_ULL(63, 32)
#define IRDMA_UDA_CQPSQ_MAV_ADDR3_S 0
-#define IRDMA_UDA_CQPSQ_MAV_ADDR3_M \
- ((u64)0xffffffff << IRDMA_UDA_CQPSQ_MAV_ADDR3_S)
-
+#define IRDMA_UDA_CQPSQ_MAV_ADDR3 GENMASK_ULL(31, 0)
#define IRDMA_UDA_CQPSQ_MAV_WQEVALID_S 63
-#define IRDMA_UDA_CQPSQ_MAV_WQEVALID_M \
- BIT_ULL(IRDMA_UDA_CQPSQ_MAV_WQEVALID_S)
-
+#define IRDMA_UDA_CQPSQ_MAV_WQEVALID BIT_ULL(63)
#define IRDMA_UDA_CQPSQ_MAV_OPCODE_S 32
-#define IRDMA_UDA_CQPSQ_MAV_OPCODE_M \
- ((u64)0x3f << IRDMA_UDA_CQPSQ_MAV_OPCODE_S)
-
+#define IRDMA_UDA_CQPSQ_MAV_OPCODE GENMASK_ULL(37, 32)
#define IRDMA_UDA_CQPSQ_MAV_DOLOOPBACKK_S 62
-#define IRDMA_UDA_CQPSQ_MAV_DOLOOPBACKK_M \
- BIT_ULL(IRDMA_UDA_CQPSQ_MAV_DOLOOPBACKK_S)
-
+#define IRDMA_UDA_CQPSQ_MAV_DOLOOPBACKK BIT_ULL(62)
#define IRDMA_UDA_CQPSQ_MAV_IPV4VALID_S 59
-#define IRDMA_UDA_CQPSQ_MAV_IPV4VALID_M \
- BIT_ULL(IRDMA_UDA_CQPSQ_MAV_IPV4VALID_S)
+#define IRDMA_UDA_CQPSQ_MAV_IPV4VALID BIT_ULL(59)
#define IRDMA_UDA_CQPSQ_MAV_AVIDX_S 0
-#define IRDMA_UDA_CQPSQ_MAV_AVIDX_M \
- ((u64)0x1ffff << IRDMA_UDA_CQPSQ_MAV_AVIDX_S)
-
+#define IRDMA_UDA_CQPSQ_MAV_AVIDX GENMASK_ULL(16, 0)
#define IRDMA_UDA_CQPSQ_MAV_INSERTVLANTAG_S 60
-#define IRDMA_UDA_CQPSQ_MAV_INSERTVLANTAG_M BIT_ULL(IRDMA_UDA_CQPSQ_MAV_INSERTVLANTAG_S)
-
-/* UDA multicast group */
-
+#define IRDMA_UDA_CQPSQ_MAV_INSERTVLANTAG BIT_ULL(60)
#define IRDMA_UDA_MGCTX_VFFLAG_S 29
-#define IRDMA_UDA_MGCTX_VFFLAG_M BIT_ULL(IRDMA_UDA_MGCTX_VFFLAG_S)
-
+#define IRDMA_UDA_MGCTX_VFFLAG BIT_ULL(29)
#define IRDMA_UDA_MGCTX_DESTPORT_S 32
-#define IRDMA_UDA_MGCTX_DESTPORT_M ((u64)0xffff << IRDMA_UDA_MGCTX_DESTPORT_S)
-
+#define IRDMA_UDA_MGCTX_DESTPORT GENMASK_ULL(47, 32)
#define IRDMA_UDA_MGCTX_VFID_S 22
-#define IRDMA_UDA_MGCTX_VFID_M ((u64)0x7f << IRDMA_UDA_MGCTX_VFID_S)
-
+#define IRDMA_UDA_MGCTX_VFID GENMASK_ULL(28, 22)
#define IRDMA_UDA_MGCTX_VALIDENT_S 31
-#define IRDMA_UDA_MGCTX_VALIDENT_M BIT_ULL(IRDMA_UDA_MGCTX_VALIDENT_S)
-
+#define IRDMA_UDA_MGCTX_VALIDENT BIT_ULL(31)
#define IRDMA_UDA_MGCTX_PFID_S 18
-#define IRDMA_UDA_MGCTX_PFID_M ((u64)0xf << IRDMA_UDA_MGCTX_PFID_S)
-
+#define IRDMA_UDA_MGCTX_PFID GENMASK_ULL(21, 18)
#define IRDMA_UDA_MGCTX_FLAGIGNOREDPORT_S 30
-#define IRDMA_UDA_MGCTX_FLAGIGNOREDPORT_M \
- BIT_ULL(IRDMA_UDA_MGCTX_FLAGIGNOREDPORT_S)
-
+#define IRDMA_UDA_MGCTX_FLAGIGNOREDPORT BIT_ULL(30)
#define IRDMA_UDA_MGCTX_QPID_S 0
-#define IRDMA_UDA_MGCTX_QPID_M ((u64)0x3ffff << IRDMA_UDA_MGCTX_QPID_S)
-
-/* multicast group create CQP command */
-
+#define IRDMA_UDA_MGCTX_QPID GENMASK_ULL(17, 0)
#define IRDMA_UDA_CQPSQ_MG_WQEVALID_S 63
-#define IRDMA_UDA_CQPSQ_MG_WQEVALID_M \
- BIT_ULL(IRDMA_UDA_CQPSQ_MG_WQEVALID_S)
-
+#define IRDMA_UDA_CQPSQ_MG_WQEVALID BIT_ULL(63)
#define IRDMA_UDA_CQPSQ_MG_OPCODE_S 32
-#define IRDMA_UDA_CQPSQ_MG_OPCODE_M ((u64)0x3f << IRDMA_UDA_CQPSQ_MG_OPCODE_S)
-
+#define IRDMA_UDA_CQPSQ_MG_OPCODE GENMASK_ULL(37, 32)
#define IRDMA_UDA_CQPSQ_MG_MGIDX_S 0
-#define IRDMA_UDA_CQPSQ_MG_MGIDX_M ((u64)0x1fff << IRDMA_UDA_CQPSQ_MG_MGIDX_S)
-
+#define IRDMA_UDA_CQPSQ_MG_MGIDX GENMASK_ULL(12, 0)
#define IRDMA_UDA_CQPSQ_MG_IPV4VALID_S 60
-#define IRDMA_UDA_CQPSQ_MG_IPV4VALID_M BIT_ULL(IRDMA_UDA_CQPSQ_MG_IPV4VALID_S)
-
+#define IRDMA_UDA_CQPSQ_MG_IPV4VALID BIT_ULL(60)
#define IRDMA_UDA_CQPSQ_MG_VLANVALID_S 59
-#define IRDMA_UDA_CQPSQ_MG_VLANVALID_M BIT_ULL(IRDMA_UDA_CQPSQ_MG_VLANVALID_S)
-
+#define IRDMA_UDA_CQPSQ_MG_VLANVALID BIT_ULL(59)
#define IRDMA_UDA_CQPSQ_MG_HMC_FCN_ID_S 0
-#define IRDMA_UDA_CQPSQ_MG_HMC_FCN_ID_M ((u64)0x3F << IRDMA_UDA_CQPSQ_MG_HMC_FCN_ID_S)
-
+#define IRDMA_UDA_CQPSQ_MG_HMC_FCN_ID GENMASK_ULL(5, 0)
#define IRDMA_UDA_CQPSQ_MG_VLANID_S 32
-#define IRDMA_UDA_CQPSQ_MG_VLANID_M ((u64)0xFFF << IRDMA_UDA_CQPSQ_MG_VLANID_S)
-
+#define IRDMA_UDA_CQPSQ_MG_VLANID GENMASK_ULL(43, 32)
#define IRDMA_UDA_CQPSQ_QS_HANDLE_S 0
-#define IRDMA_UDA_CQPSQ_QS_HANDLE_M ((u64)0x3FF << IRDMA_UDA_CQPSQ_QS_HANDLE_S)
-
-/* Quad hash table */
+#define IRDMA_UDA_CQPSQ_QS_HANDLE GENMASK_ULL(9, 0)
#define IRDMA_UDA_CQPSQ_QHASH_QPN_S 32
-#define IRDMA_UDA_CQPSQ_QHASH_QPN_M \
- ((u64)0x3ffff << IRDMA_UDA_CQPSQ_QHASH_QPN_S)
-
+#define IRDMA_UDA_CQPSQ_QHASH_QPN GENMASK_ULL(49, 32)
#define IRDMA_UDA_CQPSQ_QHASH__S 0
-#define IRDMA_UDA_CQPSQ_QHASH__M BIT_ULL(IRDMA_UDA_CQPSQ_QHASH__S)
-
+#define IRDMA_UDA_CQPSQ_QHASH_ BIT_ULL(0)
#define IRDMA_UDA_CQPSQ_QHASH_SRC_PORT_S 16
-#define IRDMA_UDA_CQPSQ_QHASH_SRC_PORT_M \
- ((u64)0xffff << IRDMA_UDA_CQPSQ_QHASH_SRC_PORT_S)
-
+#define IRDMA_UDA_CQPSQ_QHASH_SRC_PORT GENMASK_ULL(31, 16)
#define IRDMA_UDA_CQPSQ_QHASH_DEST_PORT_S 0
-#define IRDMA_UDA_CQPSQ_QHASH_DEST_PORT_M \
- ((u64)0xffff << IRDMA_UDA_CQPSQ_QHASH_DEST_PORT_S)
-
+#define IRDMA_UDA_CQPSQ_QHASH_DEST_PORT GENMASK_ULL(15, 0)
#define IRDMA_UDA_CQPSQ_QHASH_ADDR0_S 32
-#define IRDMA_UDA_CQPSQ_QHASH_ADDR0_M \
- ((u64)0xffffffff << IRDMA_UDA_CQPSQ_QHASH_ADDR0_S)
-
+#define IRDMA_UDA_CQPSQ_QHASH_ADDR0 GENMASK_ULL(63, 32)
#define IRDMA_UDA_CQPSQ_QHASH_ADDR1_S 0
-#define IRDMA_UDA_CQPSQ_QHASH_ADDR1_M \
- ((u64)0xffffffff << IRDMA_UDA_CQPSQ_QHASH_ADDR1_S)
-
+#define IRDMA_UDA_CQPSQ_QHASH_ADDR1 GENMASK_ULL(31, 0)
#define IRDMA_UDA_CQPSQ_QHASH_ADDR2_S 32
-#define IRDMA_UDA_CQPSQ_QHASH_ADDR2_M \
- ((u64)0xffffffff << IRDMA_UDA_CQPSQ_QHASH_ADDR2_S)
-
+#define IRDMA_UDA_CQPSQ_QHASH_ADDR2 GENMASK_ULL(63, 32)
#define IRDMA_UDA_CQPSQ_QHASH_ADDR3_S 0
-#define IRDMA_UDA_CQPSQ_QHASH_ADDR3_M \
- ((u64)0xffffffff << IRDMA_UDA_CQPSQ_QHASH_ADDR3_S)
-
+#define IRDMA_UDA_CQPSQ_QHASH_ADDR3 GENMASK_ULL(31, 0)
#define IRDMA_UDA_CQPSQ_QHASH_WQEVALID_S 63
-#define IRDMA_UDA_CQPSQ_QHASH_WQEVALID_M \
- BIT_ULL(IRDMA_UDA_CQPSQ_QHASH_WQEVALID_S)
-
+#define IRDMA_UDA_CQPSQ_QHASH_WQEVALID BIT_ULL(63)
#define IRDMA_UDA_CQPSQ_QHASH_OPCODE_S 32
-#define IRDMA_UDA_CQPSQ_QHASH_OPCODE_M \
- ((u64)0x3f << IRDMA_UDA_CQPSQ_QHASH_OPCODE_S)
-
+#define IRDMA_UDA_CQPSQ_QHASH_OPCODE GENMASK_ULL(37, 32)
#define IRDMA_UDA_CQPSQ_QHASH_MANAGE_S 61
-#define IRDMA_UDA_CQPSQ_QHASH_MANAGE_M \
- ((u64)0x3 << IRDMA_UDA_CQPSQ_QHASH_MANAGE_S)
-
+#define IRDMA_UDA_CQPSQ_QHASH_MANAGE GENMASK_ULL(62, 61)
#define IRDMA_UDA_CQPSQ_QHASH_IPV4VALID_S 60
-#define IRDMA_UDA_CQPSQ_QHASH_IPV4VALID_M \
- ((u64)0x1 << IRDMA_UDA_CQPSQ_QHASH_IPV4VALID_S)
-
+#define IRDMA_UDA_CQPSQ_QHASH_IPV4VALID BIT_ULL(60)
#define IRDMA_UDA_CQPSQ_QHASH_LANFWD_S 59
-#define IRDMA_UDA_CQPSQ_QHASH_LANFWD_M \
- ((u64)0x1 << IRDMA_UDA_CQPSQ_QHASH_LANFWD_S)
-
+#define IRDMA_UDA_CQPSQ_QHASH_LANFWD BIT_ULL(59)
#define IRDMA_UDA_CQPSQ_QHASH_ENTRYTYPE_S 42
-#define IRDMA_UDA_CQPSQ_QHASH_ENTRYTYPE_M \
- ((u64)0x7 << IRDMA_UDA_CQPSQ_QHASH_ENTRYTYPE_S)
+#define IRDMA_UDA_CQPSQ_QHASH_ENTRYTYPE GENMASK_ULL(44, 42)
#endif /* IRDMA_UDA_D_H */
diff --git a/sys/dev/irdma/irdma_uk.c b/sys/dev/irdma/irdma_uk.c
--- a/sys/dev/irdma/irdma_uk.c
+++ b/sys/dev/irdma/irdma_uk.c
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2021 Intel Corporation
+ * Copyright (c) 2015 - 2022 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -51,15 +51,15 @@
{
if (sge) {
set_64bit_val(wqe, offset,
- LS_64(sge->tag_off, IRDMAQPSQ_FRAG_TO));
+ FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->tag_off));
set_64bit_val(wqe, offset + IRDMA_BYTE_8,
- LS_64(valid, IRDMAQPSQ_VALID) |
- LS_64(sge->len, IRDMAQPSQ_FRAG_LEN) |
- LS_64(sge->stag, IRDMAQPSQ_FRAG_STAG));
+ FIELD_PREP(IRDMAQPSQ_VALID, valid) |
+ FIELD_PREP(IRDMAQPSQ_FRAG_LEN, sge->len) |
+ FIELD_PREP(IRDMAQPSQ_FRAG_STAG, sge->stag));
} else {
set_64bit_val(wqe, offset, 0);
set_64bit_val(wqe, offset + IRDMA_BYTE_8,
- LS_64(valid, IRDMAQPSQ_VALID));
+ FIELD_PREP(IRDMAQPSQ_VALID, valid));
}
}
@@ -76,16 +76,26 @@
{
if (sge) {
set_64bit_val(wqe, offset,
- LS_64(sge->tag_off, IRDMAQPSQ_FRAG_TO));
+ FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->tag_off));
set_64bit_val(wqe, offset + IRDMA_BYTE_8,
- LS_64(sge->len, IRDMAQPSQ_GEN1_FRAG_LEN) |
- LS_64(sge->stag, IRDMAQPSQ_GEN1_FRAG_STAG));
+ FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, sge->len) |
+ FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, sge->stag));
} else {
set_64bit_val(wqe, offset, 0);
set_64bit_val(wqe, offset + IRDMA_BYTE_8, 0);
}
}
+/**
+ * irdma_nop_hdr - Format header section of noop WQE
+ * @qp: hw qp ptr
+ */
+static inline u64 irdma_nop_hdr(struct irdma_qp_uk *qp){
+ return FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) |
+ FIELD_PREP(IRDMAQPSQ_SIGCOMPL, false) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
+}
+
/**
* irdma_nop_1 - insert a NOP wqe
* @qp: hw qp ptr
@@ -93,10 +103,8 @@
static int
irdma_nop_1(struct irdma_qp_uk *qp)
{
- u64 hdr;
__le64 *wqe;
u32 wqe_idx;
- bool signaled = false;
if (!qp->sq_ring.head)
return -EINVAL;
@@ -110,14 +118,10 @@
set_64bit_val(wqe, IRDMA_BYTE_8, 0);
set_64bit_val(wqe, IRDMA_BYTE_16, 0);
- hdr = LS_64(IRDMAQP_OP_NOP, IRDMAQPSQ_OPCODE) |
- LS_64(signaled, IRDMAQPSQ_SIGCOMPL) |
- LS_64(qp->swqe_polarity, IRDMAQPSQ_VALID);
-
/* make sure WQE is written before valid bit is set */
irdma_wmb();
- set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
+ set_64bit_val(wqe, IRDMA_BYTE_24, irdma_nop_hdr(qp));
return 0;
}
@@ -160,7 +164,7 @@
/* read the doorbell shadow area */
get_64bit_val(qp->shadow_area, IRDMA_BYTE_0, &temp);
- hw_sq_tail = (u32)RS_64(temp, IRDMA_QP_DBSA_HW_SQ_TAIL);
+ hw_sq_tail = (u32)FIELD_GET(IRDMA_QP_DBSA_HW_SQ_TAIL, temp);
sw_sq_head = IRDMA_RING_CURRENT_HEAD(qp->sq_ring);
if (sw_sq_head != qp->initial_ring.head) {
if (qp->push_dropped) {
@@ -191,7 +195,7 @@
irdma_qp_ring_push_db(struct irdma_qp_uk *qp, u32 wqe_idx)
{
set_32bit_val(qp->push_db, 0,
- LS_32(wqe_idx >> 3, IRDMA_WQEALLOC_WQE_DESC_INDEX) | qp->qp_id);
+ FIELD_PREP(IRDMA_WQEALLOC_WQE_DESC_INDEX, wqe_idx >> 3) | qp->qp_id);
qp->initial_ring.head = qp->sq_ring.head;
qp->push_mode = true;
qp->push_dropped = false;
@@ -220,31 +224,32 @@
* irdma_qp_get_next_send_wqe - pad with NOP if needed, return where next WR should go
* @qp: hw qp ptr
* @wqe_idx: return wqe index
- * @quanta: size of WR in quanta
+ * @quanta: (in/out) ptr to size of WR in quanta. Modified in case pad is needed
* @total_size: size of WR in bytes
* @info: info on WR
*/
__le64 *
irdma_qp_get_next_send_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx,
- u16 quanta, u32 total_size,
+ u16 *quanta, u32 total_size,
struct irdma_post_sq_info *info)
{
__le64 *wqe;
__le64 *wqe_0 = NULL;
u32 nop_wqe_idx;
- u16 avail_quanta;
+ u16 avail_quanta, wqe_quanta = *quanta;
u16 i;
avail_quanta = qp->uk_attrs->max_hw_sq_chunk -
(IRDMA_RING_CURRENT_HEAD(qp->sq_ring) %
qp->uk_attrs->max_hw_sq_chunk);
- if (quanta <= avail_quanta) {
+
+ if (*quanta <= avail_quanta) {
/* WR fits in current chunk */
- if (quanta > IRDMA_SQ_RING_FREE_QUANTA(qp->sq_ring))
+ if (*quanta > IRDMA_SQ_RING_FREE_QUANTA(qp->sq_ring))
return NULL;
} else {
/* Need to pad with NOP */
- if (quanta + avail_quanta >
+ if (*quanta + avail_quanta >
IRDMA_SQ_RING_FREE_QUANTA(qp->sq_ring))
return NULL;
@@ -262,17 +267,19 @@
if (!*wqe_idx)
qp->swqe_polarity = !qp->swqe_polarity;
- IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, quanta);
+ IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, *quanta);
+
+ irdma_clr_wqes(qp, *wqe_idx);
wqe = qp->sq_base[*wqe_idx].elem;
- if (qp->uk_attrs->hw_rev == IRDMA_GEN_1 && quanta == 1 &&
+ if (qp->uk_attrs->hw_rev == IRDMA_GEN_1 && wqe_quanta == 1 &&
(IRDMA_RING_CURRENT_HEAD(qp->sq_ring) & 1)) {
wqe_0 = qp->sq_base[IRDMA_RING_CURRENT_HEAD(qp->sq_ring)].elem;
- wqe_0[3] = cpu_to_le64(LS_64(!qp->swqe_polarity, IRDMAQPSQ_VALID));
+ wqe_0[3] = cpu_to_le64(FIELD_PREP(IRDMAQPSQ_VALID, !qp->swqe_polarity));
}
qp->sq_wrtrk_array[*wqe_idx].wrid = info->wr_id;
qp->sq_wrtrk_array[*wqe_idx].wr_len = total_size;
- qp->sq_wrtrk_array[*wqe_idx].quanta = quanta;
+ qp->sq_wrtrk_array[*wqe_idx].quanta = wqe_quanta;
qp->sq_wrtrk_array[*wqe_idx].signaled = info->signaled;
return wqe;
@@ -344,20 +351,17 @@
if (ret_code)
return ret_code;
- wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
- info);
+ wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, info);
if (!wqe)
return -ENOSPC;
- irdma_clr_wqes(qp, wqe_idx);
-
qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled;
set_64bit_val(wqe, IRDMA_BYTE_16,
- LS_64(op_info->rem_addr.tag_off, IRDMAQPSQ_FRAG_TO));
+ FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off));
if (info->imm_data_valid) {
set_64bit_val(wqe, IRDMA_BYTE_0,
- LS_64(info->imm_data, IRDMAQPSQ_IMMDATA));
+ FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data));
i = 0;
} else {
qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0,
@@ -382,28 +386,24 @@
++addl_frag_cnt;
}
- if (!op_info->rem_addr.stag && !total_size)
- op_info->rem_addr.stag = 0x1234;
- hdr = LS_64(op_info->rem_addr.stag, IRDMAQPSQ_REMSTAG) |
- LS_64(info->op_type, IRDMAQPSQ_OPCODE) |
- LS_64((info->imm_data_valid ? 1 : 0), IRDMAQPSQ_IMMDATAFLAG) |
- LS_64((info->report_rtt ? 1 : 0), IRDMAQPSQ_REPORTRTT) |
- LS_64(addl_frag_cnt, IRDMAQPSQ_ADDFRAGCNT) |
- LS_64((info->push_wqe ? 1 : 0), IRDMAQPSQ_PUSHWQE) |
- LS_64(read_fence, IRDMAQPSQ_READFENCE) |
- LS_64(info->local_fence, IRDMAQPSQ_LOCALFENCE) |
- LS_64(info->signaled, IRDMAQPSQ_SIGCOMPL) |
- LS_64(qp->swqe_polarity, IRDMAQPSQ_VALID);
+ hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) |
+ FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) |
+ FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, info->imm_data_valid) |
+ FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt) |
+ FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) |
+ FIELD_PREP(IRDMAQPSQ_PUSHWQE, info->push_wqe) |
+ FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) |
+ FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) |
+ FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
irdma_wmb(); /* make sure WQE is populated before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
- if (info->push_wqe) {
+ if (info->push_wqe)
irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);
- } else {
- if (post_sq)
- irdma_uk_qp_post_wr(qp);
- }
+ else if (post_sq)
+ irdma_uk_qp_post_wr(qp);
return 0;
}
@@ -443,8 +443,7 @@
if (ret_code)
return ret_code;
- wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
- info);
+ wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, info);
if (!wqe)
return -ENOSPC;
@@ -453,8 +452,6 @@
qp->ord_cnt = 0;
}
- irdma_clr_wqes(qp, wqe_idx);
-
qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled;
addl_frag_cnt = op_info->num_lo_sges > 1 ?
(op_info->num_lo_sges - 1) : 0;
@@ -478,28 +475,26 @@
++addl_frag_cnt;
}
set_64bit_val(wqe, IRDMA_BYTE_16,
- LS_64(op_info->rem_addr.tag_off, IRDMAQPSQ_FRAG_TO));
- hdr = LS_64(op_info->rem_addr.stag, IRDMAQPSQ_REMSTAG) |
- LS_64((info->report_rtt ? 1 : 0), IRDMAQPSQ_REPORTRTT) |
- LS_64(addl_frag_cnt, IRDMAQPSQ_ADDFRAGCNT) |
- LS_64((inv_stag ? IRDMAQP_OP_RDMA_READ_LOC_INV : IRDMAQP_OP_RDMA_READ),
- IRDMAQPSQ_OPCODE) |
- LS_64((info->push_wqe ? 1 : 0), IRDMAQPSQ_PUSHWQE) |
- LS_64(info->read_fence || qp->force_fence || ord_fence ? 1 : 0,
- IRDMAQPSQ_READFENCE) |
- LS_64(local_fence, IRDMAQPSQ_LOCALFENCE) |
- LS_64(info->signaled, IRDMAQPSQ_SIGCOMPL) |
- LS_64(qp->swqe_polarity, IRDMAQPSQ_VALID);
+ FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off));
+ hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) |
+ FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) |
+ FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) |
+ FIELD_PREP(IRDMAQPSQ_OPCODE,
+ (inv_stag ? IRDMAQP_OP_RDMA_READ_LOC_INV : IRDMAQP_OP_RDMA_READ)) |
+ FIELD_PREP(IRDMAQPSQ_PUSHWQE, info->push_wqe) |
+ FIELD_PREP(IRDMAQPSQ_READFENCE,
+ info->read_fence || ord_fence ? 1 : 0) |
+ FIELD_PREP(IRDMAQPSQ_LOCALFENCE, local_fence) |
+ FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
irdma_wmb(); /* make sure WQE is populated before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
- if (info->push_wqe) {
+ if (info->push_wqe)
irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);
- } else {
- if (post_sq)
- irdma_uk_qp_post_wr(qp);
- }
+ else if (post_sq)
+ irdma_uk_qp_post_wr(qp);
return 0;
}
@@ -540,21 +535,19 @@
if (ret_code)
return ret_code;
- wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
- info);
+ wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, info);
if (!wqe)
return -ENOSPC;
- irdma_clr_wqes(qp, wqe_idx);
-
read_fence |= info->read_fence;
addl_frag_cnt = frag_cnt > 1 ? (frag_cnt - 1) : 0;
if (info->imm_data_valid) {
set_64bit_val(wqe, IRDMA_BYTE_0,
- LS_64(info->imm_data, IRDMAQPSQ_IMMDATA));
+ FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data));
i = 0;
} else {
- qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0, op_info->sg_list,
+ qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0,
+ frag_cnt ? op_info->sg_list : NULL,
qp->swqe_polarity);
i = 1;
}
@@ -575,31 +568,30 @@
}
set_64bit_val(wqe, IRDMA_BYTE_16,
- LS_64(op_info->qkey, IRDMAQPSQ_DESTQKEY) |
- LS_64(op_info->dest_qp, IRDMAQPSQ_DESTQPN));
- hdr = LS_64(info->stag_to_inv, IRDMAQPSQ_REMSTAG) |
- LS_64(op_info->ah_id, IRDMAQPSQ_AHID) |
- LS_64((info->imm_data_valid ? 1 : 0), IRDMAQPSQ_IMMDATAFLAG) |
- LS_64((info->report_rtt ? 1 : 0), IRDMAQPSQ_REPORTRTT) |
- LS_64(info->op_type, IRDMAQPSQ_OPCODE) |
- LS_64(addl_frag_cnt, IRDMAQPSQ_ADDFRAGCNT) |
- LS_64((info->push_wqe ? 1 : 0), IRDMAQPSQ_PUSHWQE) |
- LS_64(read_fence, IRDMAQPSQ_READFENCE) |
- LS_64(info->local_fence, IRDMAQPSQ_LOCALFENCE) |
- LS_64(info->signaled, IRDMAQPSQ_SIGCOMPL) |
- LS_64(info->udp_hdr, IRDMAQPSQ_UDPHEADER) |
- LS_64(info->l4len, IRDMAQPSQ_L4LEN) |
- LS_64(qp->swqe_polarity, IRDMAQPSQ_VALID);
+ FIELD_PREP(IRDMAQPSQ_DESTQKEY, op_info->qkey) |
+ FIELD_PREP(IRDMAQPSQ_DESTQPN, op_info->dest_qp));
+ hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, info->stag_to_inv) |
+ FIELD_PREP(IRDMAQPSQ_AHID, op_info->ah_id) |
+ FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG,
+ (info->imm_data_valid ? 1 : 0)) |
+ FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) |
+ FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) |
+ FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) |
+ FIELD_PREP(IRDMAQPSQ_PUSHWQE, info->push_wqe) |
+ FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) |
+ FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) |
+ FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
+ FIELD_PREP(IRDMAQPSQ_UDPHEADER, info->udp_hdr) |
+ FIELD_PREP(IRDMAQPSQ_L4LEN, info->l4len) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
irdma_wmb(); /* make sure WQE is populated before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
- if (info->push_wqe) {
+ if (info->push_wqe)
irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);
- } else {
- if (post_sq)
- irdma_uk_qp_post_wr(qp);
- }
+ else if (post_sq)
+ irdma_uk_qp_post_wr(qp);
return 0;
}
@@ -615,29 +607,45 @@
{
set_64bit_val(wqe, IRDMA_BYTE_0, (uintptr_t)op_info->va);
set_64bit_val(wqe, IRDMA_BYTE_8,
- LS_64(op_info->mw_stag, IRDMAQPSQ_PARENTMRSTAG) |
- LS_64(op_info->mr_stag, IRDMAQPSQ_MWSTAG));
+ FIELD_PREP(IRDMAQPSQ_PARENTMRSTAG, op_info->mw_stag) |
+ FIELD_PREP(IRDMAQPSQ_MWSTAG, op_info->mr_stag));
set_64bit_val(wqe, IRDMA_BYTE_16, op_info->bind_len);
}
/**
* irdma_copy_inline_data_gen_1 - Copy inline data to wqe
- * @dest: pointer to wqe
- * @src: pointer to inline data
- * @len: length of inline data to copy
+ * @wqe: pointer to wqe
+ * @sge_list: table of pointers to inline data
+ * @num_sges: Total inline data length
* @polarity: compatibility parameter
*/
static void
-irdma_copy_inline_data_gen_1(u8 *dest, u8 *src, u32 len,
- u8 polarity)
+irdma_copy_inline_data_gen_1(u8 *wqe, struct irdma_sge *sge_list,
+ u32 num_sges, u8 polarity)
{
- if (len <= IRDMA_BYTE_16) {
- irdma_memcpy(dest, src, len);
- } else {
- irdma_memcpy(dest, src, IRDMA_BYTE_16);
- src += IRDMA_BYTE_16;
- dest = dest + IRDMA_BYTE_32;
- irdma_memcpy(dest, src, len - IRDMA_BYTE_16);
+ u32 quanta_bytes_remaining = 16;
+ u32 i;
+
+ for (i = 0; i < num_sges; i++) {
+ u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].tag_off;
+ u32 sge_len = sge_list[i].len;
+
+ while (sge_len) {
+ u32 bytes_copied;
+
+ bytes_copied = min(sge_len, quanta_bytes_remaining);
+ irdma_memcpy(wqe, cur_sge, bytes_copied);
+ wqe += bytes_copied;
+ cur_sge += bytes_copied;
+ quanta_bytes_remaining -= bytes_copied;
+ sge_len -= bytes_copied;
+
+ if (!quanta_bytes_remaining) {
+ /* Remaining inline bytes reside after the hdr */
+ wqe += 16;
+ quanta_bytes_remaining = 32;
+ }
+ }
}
}
@@ -662,43 +670,59 @@
{
set_64bit_val(wqe, IRDMA_BYTE_0, (uintptr_t)op_info->va);
set_64bit_val(wqe, IRDMA_BYTE_8,
- LS_64(op_info->mr_stag, IRDMAQPSQ_PARENTMRSTAG) |
- LS_64(op_info->mw_stag, IRDMAQPSQ_MWSTAG));
+ FIELD_PREP(IRDMAQPSQ_PARENTMRSTAG, op_info->mr_stag) |
+ FIELD_PREP(IRDMAQPSQ_MWSTAG, op_info->mw_stag));
set_64bit_val(wqe, IRDMA_BYTE_16, op_info->bind_len);
}
/**
* irdma_copy_inline_data - Copy inline data to wqe
- * @dest: pointer to wqe
- * @src: pointer to inline data
- * @len: length of inline data to copy
+ * @wqe: pointer to wqe
+ * @sge_list: table of pointers to inline data
+ * @num_sges: number of SGE's
* @polarity: polarity of wqe valid bit
*/
static void
-irdma_copy_inline_data(u8 *dest, u8 *src, u32 len, u8 polarity)
+irdma_copy_inline_data(u8 *wqe, struct irdma_sge *sge_list, u32 num_sges,
+ u8 polarity)
{
u8 inline_valid = polarity << IRDMA_INLINE_VALID_S;
- u32 copy_size;
-
- dest += IRDMA_BYTE_8;
- if (len <= IRDMA_BYTE_8) {
- irdma_memcpy(dest, src, len);
- return;
- }
-
- *((u64 *)dest) = *((u64 *)src);
- len -= IRDMA_BYTE_8;
- src += IRDMA_BYTE_8;
- dest += IRDMA_BYTE_24; /* point to additional 32 byte quanta */
-
- while (len) {
- copy_size = len < 31 ? len : 31;
- irdma_memcpy(dest, src, copy_size);
- *(dest + 31) = inline_valid;
- len -= copy_size;
- dest += IRDMA_BYTE_32;
- src += copy_size;
+ u32 quanta_bytes_remaining = 8;
+ u32 i;
+ bool first_quanta = true;
+
+ wqe += 8;
+
+ for (i = 0; i < num_sges; i++) {
+ u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].tag_off;
+ u32 sge_len = sge_list[i].len;
+
+ while (sge_len) {
+ u32 bytes_copied;
+
+ bytes_copied = min(sge_len, quanta_bytes_remaining);
+ irdma_memcpy(wqe, cur_sge, bytes_copied);
+ wqe += bytes_copied;
+ cur_sge += bytes_copied;
+ quanta_bytes_remaining -= bytes_copied;
+ sge_len -= bytes_copied;
+
+ if (!quanta_bytes_remaining) {
+ quanta_bytes_remaining = 31;
+
+ /* Remaining inline bytes reside after the hdr */
+ if (first_quanta) {
+ first_quanta = false;
+ wqe += 16;
+ } else {
+ *wqe = inline_valid;
+ wqe++;
+ }
+ }
+ }
}
+ if (!first_quanta && quanta_bytes_remaining < 31)
+ *(wqe + quanta_bytes_remaining) = inline_valid;
}
/**
@@ -737,59 +761,62 @@
struct irdma_post_sq_info *info, bool post_sq)
{
__le64 *wqe;
- struct irdma_inline_rdma_write *op_info;
+ struct irdma_rdma_write *op_info;
u64 hdr = 0;
u32 wqe_idx;
bool read_fence = false;
u16 quanta;
+ u32 i, total_size = 0;
info->push_wqe = qp->push_db ? true : false;
- op_info = &info->op.inline_rdma_write;
+ op_info = &info->op.rdma_write;
- if (op_info->len > qp->max_inline_data)
+ if (unlikely(qp->max_sq_frag_cnt < op_info->num_lo_sges))
return -EINVAL;
- quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(op_info->len);
- wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, op_info->len,
- info);
+ for (i = 0; i < op_info->num_lo_sges; i++)
+ total_size += op_info->lo_sg_list[i].len;
+
+ if (unlikely(total_size > qp->max_inline_data))
+ return -EINVAL;
+
+ quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(total_size);
+ wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, info);
if (!wqe)
return -ENOSPC;
- irdma_clr_wqes(qp, wqe_idx);
-
qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled;
read_fence |= info->read_fence;
set_64bit_val(wqe, IRDMA_BYTE_16,
- LS_64(op_info->rem_addr.tag_off, IRDMAQPSQ_FRAG_TO));
-
- hdr = LS_64(op_info->rem_addr.stag, IRDMAQPSQ_REMSTAG) |
- LS_64(info->op_type, IRDMAQPSQ_OPCODE) |
- LS_64(op_info->len, IRDMAQPSQ_INLINEDATALEN) |
- LS_64(info->report_rtt ? 1 : 0, IRDMAQPSQ_REPORTRTT) |
- LS_64(1, IRDMAQPSQ_INLINEDATAFLAG) |
- LS_64(info->imm_data_valid ? 1 : 0, IRDMAQPSQ_IMMDATAFLAG) |
- LS_64(info->push_wqe ? 1 : 0, IRDMAQPSQ_PUSHWQE) |
- LS_64(read_fence, IRDMAQPSQ_READFENCE) |
- LS_64(info->local_fence, IRDMAQPSQ_LOCALFENCE) |
- LS_64(info->signaled, IRDMAQPSQ_SIGCOMPL) |
- LS_64(qp->swqe_polarity, IRDMAQPSQ_VALID);
+ FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off));
+
+ hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) |
+ FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) |
+ FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, total_size) |
+ FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt ? 1 : 0) |
+ FIELD_PREP(IRDMAQPSQ_INLINEDATAFLAG, 1) |
+ FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, info->imm_data_valid ? 1 : 0) |
+ FIELD_PREP(IRDMAQPSQ_PUSHWQE, info->push_wqe ? 1 : 0) |
+ FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) |
+ FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) |
+ FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
if (info->imm_data_valid)
set_64bit_val(wqe, IRDMA_BYTE_0,
- LS_64(info->imm_data, IRDMAQPSQ_IMMDATA));
+ FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data));
+
+ qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->lo_sg_list,
+ op_info->num_lo_sges, qp->swqe_polarity);
- qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->data, op_info->len,
- qp->swqe_polarity);
irdma_wmb(); /* make sure WQE is populated before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
- if (info->push_wqe) {
+ if (info->push_wqe)
irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);
- } else {
- if (post_sq)
- irdma_uk_qp_post_wr(qp);
- }
+ else if (post_sq)
+ irdma_uk_qp_post_wr(qp);
return 0;
}
@@ -805,62 +832,65 @@
struct irdma_post_sq_info *info, bool post_sq)
{
__le64 *wqe;
- struct irdma_post_inline_send *op_info;
+ struct irdma_post_send *op_info;
u64 hdr;
u32 wqe_idx;
bool read_fence = false;
u16 quanta;
+ u32 i, total_size = 0;
info->push_wqe = qp->push_db ? true : false;
- op_info = &info->op.inline_send;
+ op_info = &info->op.send;
- if (op_info->len > qp->max_inline_data)
+ if (unlikely(qp->max_sq_frag_cnt < op_info->num_sges))
return -EINVAL;
- quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(op_info->len);
- wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, op_info->len,
- info);
+ for (i = 0; i < op_info->num_sges; i++)
+ total_size += op_info->sg_list[i].len;
+
+ if (unlikely(total_size > qp->max_inline_data))
+ return -EINVAL;
+
+ quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(total_size);
+ wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, info);
if (!wqe)
return -ENOSPC;
- irdma_clr_wqes(qp, wqe_idx);
-
set_64bit_val(wqe, IRDMA_BYTE_16,
- LS_64(op_info->qkey, IRDMAQPSQ_DESTQKEY) |
- LS_64(op_info->dest_qp, IRDMAQPSQ_DESTQPN));
+ FIELD_PREP(IRDMAQPSQ_DESTQKEY, op_info->qkey) |
+ FIELD_PREP(IRDMAQPSQ_DESTQPN, op_info->dest_qp));
read_fence |= info->read_fence;
- hdr = LS_64(info->stag_to_inv, IRDMAQPSQ_REMSTAG) |
- LS_64(op_info->ah_id, IRDMAQPSQ_AHID) |
- LS_64(info->op_type, IRDMAQPSQ_OPCODE) |
- LS_64(op_info->len, IRDMAQPSQ_INLINEDATALEN) |
- LS_64((info->imm_data_valid ? 1 : 0), IRDMAQPSQ_IMMDATAFLAG) |
- LS_64((info->report_rtt ? 1 : 0), IRDMAQPSQ_REPORTRTT) |
- LS_64(1, IRDMAQPSQ_INLINEDATAFLAG) |
- LS_64((info->push_wqe ? 1 : 0), IRDMAQPSQ_PUSHWQE) |
- LS_64(read_fence, IRDMAQPSQ_READFENCE) |
- LS_64(info->local_fence, IRDMAQPSQ_LOCALFENCE) |
- LS_64(info->signaled, IRDMAQPSQ_SIGCOMPL) |
- LS_64(info->udp_hdr, IRDMAQPSQ_UDPHEADER) |
- LS_64(info->l4len, IRDMAQPSQ_L4LEN) |
- LS_64(qp->swqe_polarity, IRDMAQPSQ_VALID);
+ hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, info->stag_to_inv) |
+ FIELD_PREP(IRDMAQPSQ_AHID, op_info->ah_id) |
+ FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) |
+ FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, total_size) |
+ FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG,
+ (info->imm_data_valid ? 1 : 0)) |
+ FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) |
+ FIELD_PREP(IRDMAQPSQ_INLINEDATAFLAG, 1) |
+ FIELD_PREP(IRDMAQPSQ_PUSHWQE, info->push_wqe) |
+ FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) |
+ FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) |
+ FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
+ FIELD_PREP(IRDMAQPSQ_UDPHEADER, info->udp_hdr) |
+ FIELD_PREP(IRDMAQPSQ_L4LEN, info->l4len) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
if (info->imm_data_valid)
set_64bit_val(wqe, IRDMA_BYTE_0,
- LS_64(info->imm_data, IRDMAQPSQ_IMMDATA));
- qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->data, op_info->len,
- qp->swqe_polarity);
+ FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data));
+ qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->sg_list,
+ op_info->num_sges, qp->swqe_polarity);
irdma_wmb(); /* make sure WQE is populated before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
- if (info->push_wqe) {
+ if (info->push_wqe)
irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);
- } else {
- if (post_sq)
- irdma_uk_qp_post_wr(qp);
- }
+ else if (post_sq)
+ irdma_uk_qp_post_wr(qp);
return 0;
}
@@ -882,41 +912,36 @@
u32 wqe_idx;
bool local_fence = false;
struct irdma_sge sge = {0};
+ u16 quanta = IRDMA_QP_WQE_MIN_QUANTA;
info->push_wqe = qp->push_db ? true : false;
op_info = &info->op.inv_local_stag;
local_fence = info->local_fence;
- wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, IRDMA_QP_WQE_MIN_QUANTA,
- 0, info);
+ wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, 0, info);
if (!wqe)
return -ENOSPC;
- irdma_clr_wqes(qp, wqe_idx);
-
sge.stag = op_info->target_stag;
qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0, &sge, 0);
set_64bit_val(wqe, IRDMA_BYTE_16, 0);
- hdr = LS_64(IRDMA_OP_TYPE_INV_STAG, IRDMAQPSQ_OPCODE) |
- LS_64((info->push_wqe ? 1 : 0), IRDMAQPSQ_PUSHWQE) |
- LS_64(info->read_fence, IRDMAQPSQ_READFENCE) |
- LS_64(local_fence, IRDMAQPSQ_LOCALFENCE) |
- LS_64(info->signaled, IRDMAQPSQ_SIGCOMPL) |
- LS_64(qp->swqe_polarity, IRDMAQPSQ_VALID);
+ hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMA_OP_TYPE_INV_STAG) |
+ FIELD_PREP(IRDMAQPSQ_PUSHWQE, info->push_wqe) |
+ FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) |
+ FIELD_PREP(IRDMAQPSQ_LOCALFENCE, local_fence) |
+ FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
irdma_wmb(); /* make sure WQE is populated before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
- if (info->push_wqe) {
- irdma_qp_push_wqe(qp, wqe, IRDMA_QP_WQE_MIN_QUANTA, wqe_idx,
- post_sq);
- } else {
- if (post_sq)
- irdma_uk_qp_post_wr(qp);
- }
+ if (info->push_wqe)
+ irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);
+ else if (post_sq)
+ irdma_uk_qp_post_wr(qp);
return 0;
}
@@ -936,44 +961,39 @@
u64 hdr;
u32 wqe_idx;
bool local_fence;
+ u16 quanta = IRDMA_QP_WQE_MIN_QUANTA;
info->push_wqe = qp->push_db ? true : false;
op_info = &info->op.bind_window;
local_fence = info->local_fence;
- wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, IRDMA_QP_WQE_MIN_QUANTA,
- 0, info);
+ wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, 0, info);
if (!wqe)
return -ENOSPC;
- irdma_clr_wqes(qp, wqe_idx);
-
qp->wqe_ops.iw_set_mw_bind_wqe(wqe, op_info);
- hdr = LS_64(IRDMA_OP_TYPE_BIND_MW, IRDMAQPSQ_OPCODE) |
- LS_64(((op_info->ena_reads << 2) | (op_info->ena_writes << 3)),
- IRDMAQPSQ_STAGRIGHTS) |
- LS_64((op_info->addressing_type == IRDMA_ADDR_TYPE_VA_BASED ? 1 : 0),
- IRDMAQPSQ_VABASEDTO) |
- LS_64((op_info->mem_window_type_1 ? 1 : 0),
- IRDMAQPSQ_MEMWINDOWTYPE) |
- LS_64((info->push_wqe ? 1 : 0), IRDMAQPSQ_PUSHWQE) |
- LS_64(info->read_fence, IRDMAQPSQ_READFENCE) |
- LS_64(local_fence, IRDMAQPSQ_LOCALFENCE) |
- LS_64(info->signaled, IRDMAQPSQ_SIGCOMPL) |
- LS_64(qp->swqe_polarity, IRDMAQPSQ_VALID);
+ hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMA_OP_TYPE_BIND_MW) |
+ FIELD_PREP(IRDMAQPSQ_STAGRIGHTS,
+ ((op_info->ena_reads << 2) | (op_info->ena_writes << 3))) |
+ FIELD_PREP(IRDMAQPSQ_VABASEDTO,
+ (op_info->addressing_type == IRDMA_ADDR_TYPE_VA_BASED ? 1 : 0)) |
+ FIELD_PREP(IRDMAQPSQ_MEMWINDOWTYPE,
+ (op_info->mem_window_type_1 ? 1 : 0)) |
+ FIELD_PREP(IRDMAQPSQ_PUSHWQE, info->push_wqe) |
+ FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) |
+ FIELD_PREP(IRDMAQPSQ_LOCALFENCE, local_fence) |
+ FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
irdma_wmb(); /* make sure WQE is populated before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
- if (info->push_wqe) {
- irdma_qp_push_wqe(qp, wqe, IRDMA_QP_WQE_MIN_QUANTA, wqe_idx,
- post_sq);
- } else {
- if (post_sq)
- irdma_uk_qp_post_wr(qp);
- }
+ if (info->push_wqe)
+ irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);
+ else if (post_sq)
+ irdma_uk_qp_post_wr(qp);
return 0;
}
@@ -1020,8 +1040,8 @@
}
set_64bit_val(wqe, IRDMA_BYTE_16, 0);
- hdr = LS_64(addl_frag_cnt, IRDMAQPSQ_ADDFRAGCNT) |
- LS_64(qp->rwqe_polarity, IRDMAQPSQ_VALID);
+ hdr = FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->rwqe_polarity);
irdma_wmb(); /* make sure WQE is populated before valid bit is set */
@@ -1061,17 +1081,17 @@
get_64bit_val(cq->shadow_area, 32, &temp_val);
- sw_cq_sel = (u16)RS_64(temp_val, IRDMA_CQ_DBSA_SW_CQ_SELECT);
+ sw_cq_sel = (u16)FIELD_GET(IRDMA_CQ_DBSA_SW_CQ_SELECT, temp_val);
sw_cq_sel += cq_cnt;
- arm_seq_num = (u8)RS_64(temp_val, IRDMA_CQ_DBSA_ARM_SEQ_NUM);
- arm_next_se = (u8)RS_64(temp_val, IRDMA_CQ_DBSA_ARM_NEXT_SE);
- arm_next = (u8)RS_64(temp_val, IRDMA_CQ_DBSA_ARM_NEXT);
+ arm_seq_num = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_SEQ_NUM, temp_val);
+ arm_next_se = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT_SE, temp_val);
+ arm_next = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT, temp_val);
- temp_val = LS_64(arm_seq_num, IRDMA_CQ_DBSA_ARM_SEQ_NUM) |
- LS_64(sw_cq_sel, IRDMA_CQ_DBSA_SW_CQ_SELECT) |
- LS_64(arm_next_se, IRDMA_CQ_DBSA_ARM_NEXT_SE) |
- LS_64(arm_next, IRDMA_CQ_DBSA_ARM_NEXT);
+ temp_val = FIELD_PREP(IRDMA_CQ_DBSA_ARM_SEQ_NUM, arm_seq_num) |
+ FIELD_PREP(IRDMA_CQ_DBSA_SW_CQ_SELECT, sw_cq_sel) |
+ FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT_SE, arm_next_se) |
+ FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT, arm_next);
set_64bit_val(cq->shadow_area, 32, temp_val);
}
@@ -1093,17 +1113,17 @@
cq->armed = true;
get_64bit_val(cq->shadow_area, IRDMA_BYTE_32, &temp_val);
- arm_seq_num = (u8)RS_64(temp_val, IRDMA_CQ_DBSA_ARM_SEQ_NUM);
+ arm_seq_num = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_SEQ_NUM, temp_val);
arm_seq_num++;
- sw_cq_sel = (u16)RS_64(temp_val, IRDMA_CQ_DBSA_SW_CQ_SELECT);
- arm_next_se = (u8)RS_64(temp_val, IRDMA_CQ_DBSA_ARM_NEXT_SE);
+ sw_cq_sel = (u16)FIELD_GET(IRDMA_CQ_DBSA_SW_CQ_SELECT, temp_val);
+ arm_next_se = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT_SE, temp_val);
arm_next_se |= 1;
if (cq_notify == IRDMA_CQ_COMPL_EVENT)
arm_next = 1;
- temp_val = LS_64(arm_seq_num, IRDMA_CQ_DBSA_ARM_SEQ_NUM) |
- LS_64(sw_cq_sel, IRDMA_CQ_DBSA_SW_CQ_SELECT) |
- LS_64(arm_next_se, IRDMA_CQ_DBSA_ARM_NEXT_SE) |
- LS_64(arm_next, IRDMA_CQ_DBSA_ARM_NEXT);
+ temp_val = FIELD_PREP(IRDMA_CQ_DBSA_ARM_SEQ_NUM, arm_seq_num) |
+ FIELD_PREP(IRDMA_CQ_DBSA_SW_CQ_SELECT, sw_cq_sel) |
+ FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT_SE, arm_next_se) |
+ FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT, arm_next);
set_64bit_val(cq->shadow_area, IRDMA_BYTE_32, temp_val);
@@ -1166,7 +1186,7 @@
__le64 *cqe;
struct irdma_qp_uk *qp;
struct irdma_ring *pring = NULL;
- u32 wqe_idx, q_type;
+ u32 wqe_idx;
int ret_code;
bool move_cq_head = true;
u8 polarity;
@@ -1179,14 +1199,14 @@
cqe = IRDMA_GET_CURRENT_CQ_ELEM(cq);
get_64bit_val(cqe, IRDMA_BYTE_24, &qword3);
- polarity = (u8)RS_64(qword3, IRDMA_CQ_VALID);
+ polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3);
if (polarity != cq->polarity)
return -ENOENT;
/* Ensure CQE contents are read after valid bit is checked */
rmb();
- ext_valid = (bool)RS_64(qword3, IRDMA_CQ_EXTCQE);
+ ext_valid = (bool)FIELD_GET(IRDMA_CQ_EXTCQE, qword3);
if (ext_valid) {
u64 qword6, qword7;
u32 peek_head;
@@ -1194,12 +1214,12 @@
if (cq->avoid_mem_cflct) {
ext_cqe = (__le64 *) ((u8 *)cqe + 32);
get_64bit_val(ext_cqe, IRDMA_BYTE_24, &qword7);
- polarity = (u8)RS_64(qword7, IRDMA_CQ_VALID);
+ polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword7);
} else {
peek_head = (cq->cq_ring.head + 1) % cq->cq_ring.size;
ext_cqe = cq->cq_base[peek_head].buf;
get_64bit_val(ext_cqe, IRDMA_BYTE_24, &qword7);
- polarity = (u8)RS_64(qword7, IRDMA_CQ_VALID);
+ polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword7);
if (!peek_head)
polarity ^= 1;
}
@@ -1209,19 +1229,19 @@
/* Ensure ext CQE contents are read after ext valid bit is checked */
rmb();
- info->imm_valid = (bool)RS_64(qword7, IRDMA_CQ_IMMVALID);
+ info->imm_valid = (bool)FIELD_GET(IRDMA_CQ_IMMVALID, qword7);
if (info->imm_valid) {
u64 qword4;
get_64bit_val(ext_cqe, IRDMA_BYTE_0, &qword4);
- info->imm_data = (u32)RS_64(qword4, IRDMA_CQ_IMMDATALOW32);
+ info->imm_data = (u32)FIELD_GET(IRDMA_CQ_IMMDATALOW32, qword4);
}
- info->ud_smac_valid = (bool)RS_64(qword7, IRDMA_CQ_UDSMACVALID);
- info->ud_vlan_valid = (bool)RS_64(qword7, IRDMA_CQ_UDVLANVALID);
+ info->ud_smac_valid = (bool)FIELD_GET(IRDMA_CQ_UDSMACVALID, qword7);
+ info->ud_vlan_valid = (bool)FIELD_GET(IRDMA_CQ_UDVLANVALID, qword7);
if (info->ud_smac_valid || info->ud_vlan_valid) {
get_64bit_val(ext_cqe, IRDMA_BYTE_16, &qword6);
if (info->ud_vlan_valid)
- info->ud_vlan = (u16)RS_64(qword6, IRDMA_CQ_UDVLAN);
+ info->ud_vlan = (u16)FIELD_GET(IRDMA_CQ_UDVLAN, qword6);
if (info->ud_smac_valid) {
info->ud_smac[5] = qword6 & 0xFF;
info->ud_smac[4] = (qword6 >> 8) & 0xFF;
@@ -1237,23 +1257,26 @@
info->ud_vlan_valid = false;
}
- q_type = (u8)RS_64(qword3, IRDMA_CQ_SQ);
- info->error = (bool)RS_64(qword3, IRDMA_CQ_ERROR);
- info->push_dropped = (bool)RS_64(qword3, IRDMACQ_PSHDROP);
- info->ipv4 = (bool)RS_64(qword3, IRDMACQ_IPV4);
+ info->q_type = (u8)FIELD_GET(IRDMA_CQ_SQ, qword3);
+ info->error = (bool)FIELD_GET(IRDMA_CQ_ERROR, qword3);
+ info->push_dropped = (bool)FIELD_GET(IRDMACQ_PSHDROP, qword3);
+ info->ipv4 = (bool)FIELD_GET(IRDMACQ_IPV4, qword3);
if (info->error) {
- info->major_err = RS_64(qword3, IRDMA_CQ_MAJERR);
- info->minor_err = RS_64(qword3, IRDMA_CQ_MINERR);
- if (info->major_err == IRDMA_FLUSH_MAJOR_ERR) {
- info->comp_status = IRDMA_COMPL_STATUS_FLUSHED;
+ info->major_err = FIELD_GET(IRDMA_CQ_MAJERR, qword3);
+ info->minor_err = FIELD_GET(IRDMA_CQ_MINERR, qword3);
+ switch (info->major_err) {
+ case IRDMA_FLUSH_MAJOR_ERR:
/* Set the min error to standard flush error code for remaining cqes */
if (info->minor_err != FLUSH_GENERAL_ERR) {
- qword3 &= ~IRDMA_CQ_MINERR_M;
- qword3 |= LS_64(FLUSH_GENERAL_ERR, IRDMA_CQ_MINERR);
+ qword3 &= ~IRDMA_CQ_MINERR;
+ qword3 |= FIELD_PREP(IRDMA_CQ_MINERR, FLUSH_GENERAL_ERR);
set_64bit_val(cqe, IRDMA_BYTE_24, qword3);
}
- } else {
+ info->comp_status = IRDMA_COMPL_STATUS_FLUSHED;
+ break;
+ default:
info->comp_status = IRDMA_COMPL_STATUS_UNKNOWN;
+ break;
}
} else {
info->comp_status = IRDMA_COMPL_STATUS_SUCCESS;
@@ -1262,22 +1285,23 @@
get_64bit_val(cqe, IRDMA_BYTE_0, &qword0);
get_64bit_val(cqe, IRDMA_BYTE_16, &qword2);
- info->tcp_seq_num_rtt = (u32)RS_64(qword0, IRDMACQ_TCPSEQNUMRTT);
- info->qp_id = (u32)RS_64(qword2, IRDMACQ_QPID);
- info->ud_src_qpn = (u32)RS_64(qword2, IRDMACQ_UDSRCQPN);
+ info->stat.raw = (u32)FIELD_GET(IRDMACQ_TCPSQN_ROCEPSN_RTT_TS, qword0);
+ info->qp_id = (u32)FIELD_GET(IRDMACQ_QPID, qword2);
+ info->ud_src_qpn = (u32)FIELD_GET(IRDMACQ_UDSRCQPN, qword2);
get_64bit_val(cqe, IRDMA_BYTE_8, &comp_ctx);
- info->solicited_event = (bool)RS_64(qword3, IRDMACQ_SOEVENT);
+ info->solicited_event = (bool)FIELD_GET(IRDMACQ_SOEVENT, qword3);
qp = (struct irdma_qp_uk *)(irdma_uintptr) comp_ctx;
if (!qp || qp->destroy_pending) {
ret_code = -EFAULT;
goto exit;
}
- wqe_idx = (u32)RS_64(qword3, IRDMA_CQ_WQEIDX);
+ wqe_idx = (u32)FIELD_GET(IRDMA_CQ_WQEIDX, qword3);
info->qp_handle = (irdma_qp_handle) (irdma_uintptr) qp;
+ info->op_type = (u8)FIELD_GET(IRDMACQ_OP, qword3);
- if (q_type == IRDMA_CQE_QTYPE_RQ) {
+ if (info->q_type == IRDMA_CQE_QTYPE_RQ) {
u32 array_idx;
ret_code = irdma_skip_duplicate_flush_cmpl(qp->rq_ring,
@@ -1310,16 +1334,11 @@
}
}
- info->bytes_xfered = (u32)RS_64(qword0, IRDMACQ_PAYLDLEN);
+ info->bytes_xfered = (u32)FIELD_GET(IRDMACQ_PAYLDLEN, qword0);
- if (info->imm_valid)
- info->op_type = IRDMA_OP_TYPE_REC_IMM;
- else
- info->op_type = IRDMA_OP_TYPE_REC;
-
- if (qword3 & IRDMACQ_STAG_M) {
+ if (qword3 & IRDMACQ_STAG) {
info->stag_invalid_set = true;
- info->inv_stag = (u32)RS_64(qword2, IRDMACQ_INVSTAG);
+ info->inv_stag = (u32)FIELD_GET(IRDMACQ_INVSTAG, qword2);
} else {
info->stag_invalid_set = false;
}
@@ -1363,11 +1382,15 @@
info->signaled = qp->sq_wrtrk_array[wqe_idx].signaled;
if (!info->comp_status)
info->bytes_xfered = qp->sq_wrtrk_array[wqe_idx].wr_len;
- info->op_type = (u8)RS_64(qword3, IRDMACQ_OP);
+ info->op_type = (u8)FIELD_GET(IRDMACQ_OP, qword3);
IRDMA_RING_SET_TAIL(qp->sq_ring,
wqe_idx + qp->sq_wrtrk_array[wqe_idx].quanta);
} else {
+ unsigned long flags;
+
+ spin_lock_irqsave(qp->lock, flags);
if (!IRDMA_RING_MORE_WORK(qp->sq_ring)) {
+ spin_unlock_irqrestore(qp->lock, flags);
ret_code = -ENOENT;
goto exit;
}
@@ -1375,27 +1398,29 @@
do {
__le64 *sw_wqe;
u64 wqe_qword;
- u8 op_type;
u32 tail;
tail = qp->sq_ring.tail;
sw_wqe = qp->sq_base[tail].elem;
get_64bit_val(sw_wqe, IRDMA_BYTE_24,
&wqe_qword);
- op_type = (u8)RS_64(wqe_qword, IRDMAQPSQ_OPCODE);
- info->op_type = op_type;
+ info->op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, wqe_qword);
IRDMA_RING_SET_TAIL(qp->sq_ring,
tail + qp->sq_wrtrk_array[tail].quanta);
- if (op_type != IRDMAQP_OP_NOP) {
+ if (info->op_type != IRDMAQP_OP_NOP) {
info->wr_id = qp->sq_wrtrk_array[tail].wrid;
info->signaled = qp->sq_wrtrk_array[tail].signaled;
info->bytes_xfered = qp->sq_wrtrk_array[tail].wr_len;
break;
}
} while (1);
+
+ if (info->op_type == IRDMA_OP_TYPE_BIND_MW && info->minor_err == FLUSH_PROT_ERR)
+ info->minor_err = FLUSH_MW_BIND_ERR;
qp->sq_flush_seen = true;
if (!IRDMA_RING_MORE_WORK(qp->sq_ring))
qp->sq_flush_complete = true;
+ spin_unlock_irqrestore(qp->lock, flags);
}
pring = &qp->sq_ring;
}
@@ -1403,9 +1428,10 @@
ret_code = 0;
exit:
- if (!ret_code && info->comp_status == IRDMA_COMPL_STATUS_FLUSHED)
+ if (!ret_code && info->comp_status == IRDMA_COMPL_STATUS_FLUSHED) {
if (pring && IRDMA_RING_MORE_WORK(*pring))
move_cq_head = false;
+ }
if (move_cq_head) {
IRDMA_RING_MOVE_HEAD_NOCHECK(cq->cq_ring);
@@ -1424,8 +1450,8 @@
set_64bit_val(cq->shadow_area, IRDMA_BYTE_0,
IRDMA_RING_CURRENT_HEAD(cq->cq_ring));
} else {
- qword3 &= ~IRDMA_CQ_WQEIDX_M;
- qword3 |= LS_64(pring->tail, IRDMA_CQ_WQEIDX);
+ qword3 &= ~IRDMA_CQ_WQEIDX;
+ qword3 |= FIELD_PREP(IRDMA_CQ_WQEIDX, pring->tail);
set_64bit_val(cqe, IRDMA_BYTE_24, qword3);
}
@@ -1433,11 +1459,11 @@
}
/**
- * irdma_qp_round_up - return round up qp wq depth
+ * irdma_round_up_wq - return round up qp wq depth
* @wqdepth: wq depth in quanta to round up
*/
static int
-irdma_qp_round_up(u32 wqdepth)
+irdma_round_up_wq(u32 wqdepth)
{
int scount = 1;
@@ -1481,34 +1507,34 @@
}
/*
- * irdma_get_sqdepth - get SQ depth (quanta) @max_hw_wq_quanta: HW SQ size limit @sq_size: SQ size @shift: shift which
+ * irdma_get_sqdepth - get SQ depth (quanta) @uk_attrs: qp HW attributes @sq_size: SQ size @shift: shift which
* determines size of WQE @sqdepth: depth of SQ
*/
int
-irdma_get_sqdepth(u32 max_hw_wq_quanta, u32 sq_size, u8 shift, u32 *sqdepth)
+irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift, u32 *sqdepth)
{
- *sqdepth = irdma_qp_round_up((sq_size << shift) + IRDMA_SQ_RSVD);
+ *sqdepth = irdma_round_up_wq((sq_size << shift) + IRDMA_SQ_RSVD);
- if (*sqdepth < (IRDMA_QP_SW_MIN_WQSIZE << shift))
- *sqdepth = IRDMA_QP_SW_MIN_WQSIZE << shift;
- else if (*sqdepth > max_hw_wq_quanta)
+ if (*sqdepth < ((u32)uk_attrs->min_hw_wq_size << shift))
+ *sqdepth = uk_attrs->min_hw_wq_size << shift;
+ else if (*sqdepth > uk_attrs->max_hw_wq_quanta)
return -EINVAL;
return 0;
}
/*
- * irdma_get_rqdepth - get RQ/SRQ depth (quanta) @max_hw_rq_quanta: HW RQ/SRQ size limit @rq_size: RQ/SRQ size @shift:
- * shift which determines size of WQE @rqdepth: depth of RQ/SRQ
+ * irdma_get_rqdepth - get RQ depth (quanta) @uk_attrs: qp HW attributes @rq_size: SRQ size @shift: shift which
+ * determines size of WQE @rqdepth: depth of RQ/SRQ
*/
int
-irdma_get_rqdepth(u32 max_hw_rq_quanta, u32 rq_size, u8 shift, u32 *rqdepth)
+irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift, u32 *rqdepth)
{
- *rqdepth = irdma_qp_round_up((rq_size << shift) + IRDMA_RQ_RSVD);
+ *rqdepth = irdma_round_up_wq((rq_size << shift) + IRDMA_RQ_RSVD);
- if (*rqdepth < (IRDMA_QP_SW_MIN_WQSIZE << shift))
- *rqdepth = IRDMA_QP_SW_MIN_WQSIZE << shift;
- else if (*rqdepth > max_hw_rq_quanta)
+ if (*rqdepth < ((u32)uk_attrs->min_hw_wq_size << shift))
+ *rqdepth = uk_attrs->min_hw_wq_size << shift;
+ else if (*rqdepth > uk_attrs->max_hw_rq_quanta)
return -EINVAL;
return 0;
@@ -1549,6 +1575,80 @@
IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->initial_ring, move_cnt);
}
+/**
+ * irdma_uk_calc_shift_wq - calculate WQE shift for both SQ and RQ
+ * @ukinfo: qp initialization info
+ * @sq_shift: Returns shift of SQ
+ * @rq_shift: Returns shift of RQ
+ */
+void
+irdma_uk_calc_shift_wq(struct irdma_qp_uk_init_info *ukinfo, u8 *sq_shift,
+ u8 *rq_shift)
+{
+ bool imm_support = ukinfo->uk_attrs->hw_rev >= IRDMA_GEN_2 ? true : false;
+
+ irdma_get_wqe_shift(ukinfo->uk_attrs,
+ imm_support ? ukinfo->max_sq_frag_cnt + 1 :
+ ukinfo->max_sq_frag_cnt,
+ ukinfo->max_inline_data, sq_shift);
+
+ irdma_get_wqe_shift(ukinfo->uk_attrs, ukinfo->max_rq_frag_cnt, 0,
+ rq_shift);
+
+ if (ukinfo->uk_attrs->hw_rev == IRDMA_GEN_1) {
+ if (ukinfo->abi_ver > 4)
+ *rq_shift = IRDMA_MAX_RQ_WQE_SHIFT_GEN1;
+ }
+}
+
+/**
+ * irdma_uk_calc_depth_shift_sq - calculate depth and shift for SQ size.
+ * @ukinfo: qp initialization info
+ * @sq_depth: Returns depth of SQ
+ * @sq_shift: Returns shift of SQ
+ */
+int
+irdma_uk_calc_depth_shift_sq(struct irdma_qp_uk_init_info *ukinfo,
+ u32 *sq_depth, u8 *sq_shift)
+{
+ bool imm_support = ukinfo->uk_attrs->hw_rev >= IRDMA_GEN_2 ? true : false;
+ int status;
+ irdma_get_wqe_shift(ukinfo->uk_attrs,
+ imm_support ? ukinfo->max_sq_frag_cnt + 1 :
+ ukinfo->max_sq_frag_cnt,
+ ukinfo->max_inline_data, sq_shift);
+ status = irdma_get_sqdepth(ukinfo->uk_attrs, ukinfo->sq_size,
+ *sq_shift, sq_depth);
+
+ return status;
+}
+
+/**
+ * irdma_uk_calc_depth_shift_rq - calculate depth and shift for RQ size.
+ * @ukinfo: qp initialization info
+ * @rq_depth: Returns depth of RQ
+ * @rq_shift: Returns shift of RQ
+ */
+int
+irdma_uk_calc_depth_shift_rq(struct irdma_qp_uk_init_info *ukinfo,
+ u32 *rq_depth, u8 *rq_shift)
+{
+ int status;
+
+ irdma_get_wqe_shift(ukinfo->uk_attrs, ukinfo->max_rq_frag_cnt, 0,
+ rq_shift);
+
+ if (ukinfo->uk_attrs->hw_rev == IRDMA_GEN_1) {
+ if (ukinfo->abi_ver > 4)
+ *rq_shift = IRDMA_MAX_RQ_WQE_SHIFT_GEN1;
+ }
+
+ status = irdma_get_rqdepth(ukinfo->uk_attrs, ukinfo->rq_size,
+ *rq_shift, rq_depth);
+
+ return status;
+}
+
/**
* irdma_uk_qp_init - initialize shared qp
* @qp: hw qp (user and kernel)
@@ -1564,23 +1664,12 @@
{
int ret_code = 0;
u32 sq_ring_size;
- u8 sqshift, rqshift;
qp->uk_attrs = info->uk_attrs;
if (info->max_sq_frag_cnt > qp->uk_attrs->max_hw_wq_frags ||
info->max_rq_frag_cnt > qp->uk_attrs->max_hw_wq_frags)
return -EINVAL;
- irdma_get_wqe_shift(qp->uk_attrs, info->max_rq_frag_cnt, 0, &rqshift);
- if (qp->uk_attrs->hw_rev == IRDMA_GEN_1) {
- irdma_get_wqe_shift(qp->uk_attrs, info->max_sq_frag_cnt,
- info->max_inline_data, &sqshift);
- if (info->abi_ver > 4)
- rqshift = IRDMA_MAX_RQ_WQE_SHIFT_GEN1;
- } else {
- irdma_get_wqe_shift(qp->uk_attrs, info->max_sq_frag_cnt + 1,
- info->max_inline_data, &sqshift);
- }
qp->qp_caps = info->qp_caps;
qp->sq_base = info->sq;
qp->rq_base = info->rq;
@@ -1596,7 +1685,7 @@
qp->sq_size = info->sq_size;
qp->push_mode = false;
qp->max_sq_frag_cnt = info->max_sq_frag_cnt;
- sq_ring_size = qp->sq_size << sqshift;
+ sq_ring_size = qp->sq_size << info->sq_shift;
IRDMA_RING_INIT(qp->sq_ring, sq_ring_size);
IRDMA_RING_INIT(qp->initial_ring, sq_ring_size);
if (info->first_sq_wq) {
@@ -1611,9 +1700,9 @@
qp->rq_size = info->rq_size;
qp->max_rq_frag_cnt = info->max_rq_frag_cnt;
qp->max_inline_data = info->max_inline_data;
- qp->rq_wqe_size = rqshift;
+ qp->rq_wqe_size = info->rq_shift;
IRDMA_RING_INIT(qp->rq_ring, qp->rq_size);
- qp->rq_wqe_size_multiplier = 1 << rqshift;
+ qp->rq_wqe_size_multiplier = 1 << info->rq_shift;
if (qp->uk_attrs->hw_rev == IRDMA_GEN_1)
qp->wqe_ops = iw_wqe_uk_ops_gen_1;
else
@@ -1663,7 +1752,7 @@
else
cqe = cq->cq_base[cq_head].buf;
get_64bit_val(cqe, IRDMA_BYTE_24, &qword3);
- polarity = (u8)RS_64(qword3, IRDMA_CQ_VALID);
+ polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3);
if (polarity != temp)
break;
@@ -1693,28 +1782,29 @@
u64 hdr;
u32 wqe_idx;
struct irdma_post_sq_info info = {0};
+ u16 quanta = IRDMA_QP_WQE_MIN_QUANTA;
- info.push_wqe = false;
+ info.push_wqe = qp->push_db ? true : false;
info.wr_id = wr_id;
- wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, IRDMA_QP_WQE_MIN_QUANTA,
- 0, &info);
+ wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, 0, &info);
if (!wqe)
return -ENOSPC;
- irdma_clr_wqes(qp, wqe_idx);
-
set_64bit_val(wqe, IRDMA_BYTE_0, 0);
set_64bit_val(wqe, IRDMA_BYTE_8, 0);
set_64bit_val(wqe, IRDMA_BYTE_16, 0);
- hdr = LS_64(IRDMAQP_OP_NOP, IRDMAQPSQ_OPCODE) |
- LS_64(signaled, IRDMAQPSQ_SIGCOMPL) |
- LS_64(qp->swqe_polarity, IRDMAQPSQ_VALID);
+ hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) |
+ FIELD_PREP(IRDMAQPSQ_SIGCOMPL, signaled) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
irdma_wmb(); /* make sure WQE is populated before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
- if (post_sq)
+
+ if (info.push_wqe)
+ irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);
+ else if (post_sq)
irdma_uk_qp_post_wr(qp);
return 0;
diff --git a/sys/dev/irdma/irdma_user.h b/sys/dev/irdma/irdma_user.h
--- a/sys/dev/irdma/irdma_user.h
+++ b/sys/dev/irdma/irdma_user.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2021 Intel Corporation
+ * Copyright (c) 2015 - 2022 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -51,7 +51,7 @@
#define irdma_address_list u64 *
#define irdma_sgl struct irdma_sge *
-#define IRDMA_MAX_MR_SIZE 0x200000000000ULL
+#define IRDMA_MAX_MR_SIZE 0x200000000000ULL
#define IRDMA_ACCESS_FLAGS_LOCALREAD 0x01
#define IRDMA_ACCESS_FLAGS_LOCALWRITE 0x02
@@ -78,7 +78,97 @@
#define IRDMA_OP_TYPE_REC 0x3e
#define IRDMA_OP_TYPE_REC_IMM 0x3f
-#define IRDMA_FLUSH_MAJOR_ERR 1
+#define IRDMA_FLUSH_MAJOR_ERR 1
+#define IRDMA_SRQFLUSH_RSVD_MAJOR_ERR 0xfffe
+
+/* Async Events codes */
+#define IRDMA_AE_AMP_UNALLOCATED_STAG 0x0102
+#define IRDMA_AE_AMP_INVALID_STAG 0x0103
+#define IRDMA_AE_AMP_BAD_QP 0x0104
+#define IRDMA_AE_AMP_BAD_PD 0x0105
+#define IRDMA_AE_AMP_BAD_STAG_KEY 0x0106
+#define IRDMA_AE_AMP_BAD_STAG_INDEX 0x0107
+#define IRDMA_AE_AMP_BOUNDS_VIOLATION 0x0108
+#define IRDMA_AE_AMP_RIGHTS_VIOLATION 0x0109
+#define IRDMA_AE_AMP_TO_WRAP 0x010a
+#define IRDMA_AE_AMP_FASTREG_VALID_STAG 0x010c
+#define IRDMA_AE_AMP_FASTREG_MW_STAG 0x010d
+#define IRDMA_AE_AMP_FASTREG_INVALID_RIGHTS 0x010e
+#define IRDMA_AE_AMP_FASTREG_INVALID_LENGTH 0x0110
+#define IRDMA_AE_AMP_INVALIDATE_SHARED 0x0111
+#define IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS 0x0112
+#define IRDMA_AE_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS 0x0113
+#define IRDMA_AE_AMP_MWBIND_VALID_STAG 0x0114
+#define IRDMA_AE_AMP_MWBIND_OF_MR_STAG 0x0115
+#define IRDMA_AE_AMP_MWBIND_TO_ZERO_BASED_STAG 0x0116
+#define IRDMA_AE_AMP_MWBIND_TO_MW_STAG 0x0117
+#define IRDMA_AE_AMP_MWBIND_INVALID_RIGHTS 0x0118
+#define IRDMA_AE_AMP_MWBIND_INVALID_BOUNDS 0x0119
+#define IRDMA_AE_AMP_MWBIND_TO_INVALID_PARENT 0x011a
+#define IRDMA_AE_AMP_MWBIND_BIND_DISABLED 0x011b
+#define IRDMA_AE_PRIV_OPERATION_DENIED 0x011c
+#define IRDMA_AE_AMP_INVALIDATE_TYPE1_MW 0x011d
+#define IRDMA_AE_AMP_MWBIND_ZERO_BASED_TYPE1_MW 0x011e
+#define IRDMA_AE_AMP_FASTREG_INVALID_PBL_HPS_CFG 0x011f
+#define IRDMA_AE_AMP_MWBIND_WRONG_TYPE 0x0120
+#define IRDMA_AE_AMP_FASTREG_PBLE_MISMATCH 0x0121
+#define IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG 0x0132
+#define IRDMA_AE_UDA_XMIT_BAD_PD 0x0133
+#define IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT 0x0134
+#define IRDMA_AE_UDA_L4LEN_INVALID 0x0135
+#define IRDMA_AE_BAD_CLOSE 0x0201
+#define IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE 0x0202
+#define IRDMA_AE_CQ_OPERATION_ERROR 0x0203
+#define IRDMA_AE_RDMA_READ_WHILE_ORD_ZERO 0x0205
+#define IRDMA_AE_STAG_ZERO_INVALID 0x0206
+#define IRDMA_AE_IB_RREQ_AND_Q1_FULL 0x0207
+#define IRDMA_AE_IB_INVALID_REQUEST 0x0208
+#define IRDMA_AE_WQE_UNEXPECTED_OPCODE 0x020a
+#define IRDMA_AE_WQE_INVALID_PARAMETER 0x020b
+#define IRDMA_AE_WQE_INVALID_FRAG_DATA 0x020c
+#define IRDMA_AE_IB_REMOTE_ACCESS_ERROR 0x020d
+#define IRDMA_AE_IB_REMOTE_OP_ERROR 0x020e
+#define IRDMA_AE_WQE_LSMM_TOO_LONG 0x0220
+#define IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN 0x0301
+#define IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER 0x0303
+#define IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION 0x0304
+#define IRDMA_AE_DDP_UBE_INVALID_MO 0x0305
+#define IRDMA_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE 0x0306
+#define IRDMA_AE_DDP_UBE_INVALID_QN 0x0307
+#define IRDMA_AE_DDP_NO_L_BIT 0x0308
+#define IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION 0x0311
+#define IRDMA_AE_RDMAP_ROE_UNEXPECTED_OPCODE 0x0312
+#define IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST 0x0313
+#define IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP 0x0314
+#define IRDMA_AE_ROCE_RSP_LENGTH_ERROR 0x0316
+#define IRDMA_AE_ROCE_EMPTY_MCG 0x0380
+#define IRDMA_AE_ROCE_BAD_MC_IP_ADDR 0x0381
+#define IRDMA_AE_ROCE_BAD_MC_QPID 0x0382
+#define IRDMA_AE_MCG_QP_PROTOCOL_MISMATCH 0x0383
+#define IRDMA_AE_INVALID_ARP_ENTRY 0x0401
+#define IRDMA_AE_INVALID_TCP_OPTION_RCVD 0x0402
+#define IRDMA_AE_STALE_ARP_ENTRY 0x0403
+#define IRDMA_AE_INVALID_AH_ENTRY 0x0406
+#define IRDMA_AE_LLP_CLOSE_COMPLETE 0x0501
+#define IRDMA_AE_LLP_CONNECTION_RESET 0x0502
+#define IRDMA_AE_LLP_FIN_RECEIVED 0x0503
+#define IRDMA_AE_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH 0x0504
+#define IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR 0x0505
+#define IRDMA_AE_LLP_SEGMENT_TOO_SMALL 0x0507
+#define IRDMA_AE_LLP_SYN_RECEIVED 0x0508
+#define IRDMA_AE_LLP_TERMINATE_RECEIVED 0x0509
+#define IRDMA_AE_LLP_TOO_MANY_RETRIES 0x050a
+#define IRDMA_AE_LLP_TOO_MANY_KEEPALIVE_RETRIES 0x050b
+#define IRDMA_AE_LLP_DOUBT_REACHABILITY 0x050c
+#define IRDMA_AE_LLP_CONNECTION_ESTABLISHED 0x050e
+#define IRDMA_AE_RESOURCE_EXHAUSTION 0x0520
+#define IRDMA_AE_RESET_SENT 0x0601
+#define IRDMA_AE_TERMINATE_SENT 0x0602
+#define IRDMA_AE_RESET_NOT_SENT 0x0603
+#define IRDMA_AE_LCE_QP_CATASTROPHIC 0x0700
+#define IRDMA_AE_LCE_FUNCTION_CATASTROPHIC 0x0701
+#define IRDMA_AE_LCE_CQ_CATASTROPHIC 0x0702
+#define IRDMA_AE_QP_SUSPEND_COMPLETE 0x0900
enum irdma_device_caps_const {
IRDMA_WQE_SIZE = 4,
@@ -119,6 +209,7 @@
IRDMA_Q2_BUF_SIZE = 256,
IRDMA_QP_CTX_SIZE = 256,
IRDMA_MAX_PDS = 262144,
+ IRDMA_MIN_WQ_SIZE_GEN2 = 8,
};
enum irdma_addressing_type {
@@ -135,9 +226,15 @@
FLUSH_REM_OP_ERR,
FLUSH_LOC_LEN_ERR,
FLUSH_FATAL_ERR,
+ FLUSH_RETRY_EXC_ERR,
FLUSH_MW_BIND_ERR,
FLUSH_REM_INV_REQ_ERR,
- FLUSH_RETRY_EXC_ERR,
+};
+
+enum irdma_qp_event_type {
+ IRDMA_QP_EVENT_CATASTROPHIC,
+ IRDMA_QP_EVENT_ACCESS_ERR,
+ IRDMA_QP_EVENT_REQ_ERR,
};
enum irdma_cmpl_status {
@@ -193,7 +290,7 @@
struct irdma_ring {
volatile u32 head;
- volatile u32 tail;
+ volatile u32 tail; /* effective tail */
u32 size;
};
@@ -213,14 +310,6 @@
u32 ah_id;
};
-struct irdma_post_inline_send {
- void *data;
- u32 len;
- u32 qkey;
- u32 dest_qp;
- u32 ah_id;
-};
-
struct irdma_post_rq_info {
u64 wr_id;
irdma_sgl sg_list;
@@ -233,12 +322,6 @@
struct irdma_sge rem_addr;
};
-struct irdma_inline_rdma_write {
- void *data;
- u32 len;
- struct irdma_sge rem_addr;
-};
-
struct irdma_rdma_read {
irdma_sgl lo_sg_list;
u32 num_lo_sges;
@@ -281,8 +364,6 @@
struct irdma_rdma_read rdma_read;
struct irdma_bind_window bind_window;
struct irdma_inv_local_stag inv_local_stag;
- struct irdma_inline_rdma_write inline_rdma_write;
- struct irdma_post_inline_send inline_send;
} op;
};
@@ -290,7 +371,6 @@
u64 wr_id;
irdma_qp_handle qp_handle;
u32 bytes_xfered;
- u32 tcp_seq_num_rtt;
u32 qp_id;
u32 ud_src_qpn;
u32 imm_data;
@@ -301,6 +381,7 @@
u16 ud_vlan;
u8 ud_smac[6];
u8 op_type;
+ u8 q_type;
bool stag_invalid_set:1; /* or L_R_Key set */
bool push_dropped:1;
bool error:1;
@@ -310,6 +391,17 @@
bool ud_smac_valid:1;
bool imm_valid:1;
bool signaled:1;
+ union {
+ u32 tcp_sqn;
+ u32 roce_psn;
+ u32 rtt;
+ u32 raw;
+ } stat;
+};
+
+struct qp_err_code {
+ enum irdma_flush_opcode flush_code;
+ enum irdma_qp_event_type event_type;
};
int irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp,
@@ -334,7 +426,7 @@
bool post_sq);
struct irdma_wqe_uk_ops {
- void (*iw_copy_inline_data)(u8 *dest, u8 *src, u32 len, u8 polarity);
+ void (*iw_copy_inline_data)(u8 *dest, struct irdma_sge *sge_list, u32 num_sges, u8 polarity);
u16 (*iw_inline_data_size_to_quanta)(u32 data_size);
void (*iw_set_fragment)(__le64 *wqe, u32 offset, struct irdma_sge *sge,
u8 valid);
@@ -352,6 +444,12 @@
struct irdma_cq_uk_init_info *info);
int irdma_uk_qp_init(struct irdma_qp_uk *qp,
struct irdma_qp_uk_init_info *info);
+void irdma_uk_calc_shift_wq(struct irdma_qp_uk_init_info *ukinfo, u8 *sq_shift,
+ u8 *rq_shift);
+int irdma_uk_calc_depth_shift_sq(struct irdma_qp_uk_init_info *ukinfo,
+ u32 *sq_depth, u8 *sq_shift);
+int irdma_uk_calc_depth_shift_rq(struct irdma_qp_uk_init_info *ukinfo,
+ u32 *rq_depth, u8 *rq_shift);
struct irdma_sq_uk_wr_trk_info {
u64 wrid;
u32 wr_len;
@@ -405,7 +503,6 @@
bool destroy_pending:1; /* Indicates the QP is being destroyed */
void *back_qp;
spinlock_t *lock;
- bool force_fence;
u8 dbg_rq_flushed;
u16 ord_cnt;
u8 sq_flush_seen;
@@ -442,8 +539,12 @@
u32 max_sq_frag_cnt;
u32 max_rq_frag_cnt;
u32 max_inline_data;
+ u32 sq_depth;
+ u32 rq_depth;
u8 first_sq_wq;
u8 type;
+ u8 sq_shift;
+ u8 rq_shift;
u8 rd_fence_rate;
int abi_ver;
bool legacy_mode;
@@ -460,7 +561,7 @@
};
__le64 *irdma_qp_get_next_send_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx,
- u16 quanta, u32 total_size,
+ u16 *quanta, u32 total_size,
struct irdma_post_sq_info *info);
__le64 *irdma_qp_get_next_recv_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx);
int irdma_uk_clean_cq(void *q, struct irdma_cq_uk *cq);
@@ -469,9 +570,81 @@
int irdma_fragcnt_to_wqesize_rq(u32 frag_cnt, u16 *wqe_size);
void irdma_get_wqe_shift(struct irdma_uk_attrs *uk_attrs, u32 sge,
u32 inline_data, u8 *shift);
-int irdma_get_sqdepth(u32 max_hw_wq_quanta, u32 sq_size, u8 shift, u32 *wqdepth);
-int irdma_get_rqdepth(u32 max_hw_rq_quanta, u32 rq_size, u8 shift, u32 *wqdepth);
+int irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift, u32 *sqdepth);
+int irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift, u32 *rqdepth);
+int irdma_get_srqdepth(struct irdma_uk_attrs *uk_attrs, u32 srq_size, u8 shift, u32 *srqdepth);
void irdma_qp_push_wqe(struct irdma_qp_uk *qp, __le64 *wqe, u16 quanta,
u32 wqe_idx, bool post_sq);
void irdma_clr_wqes(struct irdma_qp_uk *qp, u32 qp_wqe_idx);
+
+static inline struct qp_err_code irdma_ae_to_qp_err_code(u16 ae_id)
+{
+ struct qp_err_code qp_err = { 0 };
+
+ switch (ae_id) {
+ case IRDMA_AE_AMP_BOUNDS_VIOLATION:
+ case IRDMA_AE_AMP_INVALID_STAG:
+ case IRDMA_AE_AMP_RIGHTS_VIOLATION:
+ case IRDMA_AE_AMP_UNALLOCATED_STAG:
+ case IRDMA_AE_AMP_BAD_PD:
+ case IRDMA_AE_AMP_BAD_QP:
+ case IRDMA_AE_AMP_BAD_STAG_KEY:
+ case IRDMA_AE_AMP_BAD_STAG_INDEX:
+ case IRDMA_AE_AMP_TO_WRAP:
+ case IRDMA_AE_PRIV_OPERATION_DENIED:
+ qp_err.flush_code = FLUSH_PROT_ERR;
+ qp_err.event_type = IRDMA_QP_EVENT_ACCESS_ERR;
+ break;
+ case IRDMA_AE_UDA_XMIT_BAD_PD:
+ case IRDMA_AE_WQE_UNEXPECTED_OPCODE:
+ qp_err.flush_code = FLUSH_LOC_QP_OP_ERR;
+ qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
+ break;
+ case IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT:
+ case IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG:
+ case IRDMA_AE_UDA_L4LEN_INVALID:
+ case IRDMA_AE_DDP_UBE_INVALID_MO:
+ case IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
+ qp_err.flush_code = FLUSH_LOC_LEN_ERR;
+ qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
+ break;
+ case IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
+ case IRDMA_AE_IB_REMOTE_ACCESS_ERROR:
+ qp_err.flush_code = FLUSH_REM_ACCESS_ERR;
+ qp_err.event_type = IRDMA_QP_EVENT_ACCESS_ERR;
+ break;
+ case IRDMA_AE_AMP_MWBIND_INVALID_RIGHTS:
+ case IRDMA_AE_AMP_MWBIND_BIND_DISABLED:
+ case IRDMA_AE_AMP_MWBIND_INVALID_BOUNDS:
+ case IRDMA_AE_AMP_MWBIND_VALID_STAG:
+ qp_err.flush_code = FLUSH_MW_BIND_ERR;
+ qp_err.event_type = IRDMA_QP_EVENT_ACCESS_ERR;
+ break;
+ case IRDMA_AE_LLP_TOO_MANY_RETRIES:
+ qp_err.flush_code = FLUSH_RETRY_EXC_ERR;
+ qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
+ break;
+ case IRDMA_AE_IB_INVALID_REQUEST:
+ qp_err.flush_code = FLUSH_REM_INV_REQ_ERR;
+ qp_err.event_type = IRDMA_QP_EVENT_REQ_ERR;
+ break;
+ case IRDMA_AE_LLP_SEGMENT_TOO_SMALL:
+ case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR:
+ case IRDMA_AE_ROCE_RSP_LENGTH_ERROR:
+ case IRDMA_AE_IB_REMOTE_OP_ERROR:
+ qp_err.flush_code = FLUSH_REM_OP_ERR;
+ qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
+ break;
+ case IRDMA_AE_LCE_QP_CATASTROPHIC:
+ qp_err.flush_code = FLUSH_FATAL_ERR;
+ qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
+ break;
+ default:
+ qp_err.flush_code = FLUSH_GENERAL_ERR;
+ qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
+ break;
+ }
+
+ return qp_err;
+}
#endif /* IRDMA_USER_H */
diff --git a/sys/dev/irdma/irdma_utils.c b/sys/dev/irdma/irdma_utils.c
--- a/sys/dev/irdma/irdma_utils.c
+++ b/sys/dev/irdma/irdma_utils.c
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2021 Intel Corporation
+ * Copyright (c) 2015 - 2022 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -46,7 +46,7 @@
* @action: modify, delete or add
*/
int
-irdma_arp_table(struct irdma_pci_f *rf, u32 *ip_addr, u8 *mac_addr,
+irdma_arp_table(struct irdma_pci_f *rf, u32 *ip_addr, const u8 *mac_addr,
u32 action)
{
unsigned long flags;
@@ -110,7 +110,7 @@
* @mac: MAC address
*/
int
-irdma_add_arp(struct irdma_pci_f *rf, u32 *ip, u8 *mac)
+irdma_add_arp(struct irdma_pci_f *rf, u32 *ip, const u8 *mac)
{
int arpidx;
@@ -128,6 +128,57 @@
return irdma_arp_table(rf, ip, NULL, IRDMA_ARP_RESOLVE);
}
+/**
+ * irdma_netdevice_event - system notifier for netdev events
+ * @notifier: not used
+ * @event: event for notifier
+ * @ptr: netdev
+ */
+int
+irdma_netdevice_event(struct notifier_block *notifier, unsigned long event,
+ void *ptr)
+{
+ struct irdma_device *iwdev;
+ struct ifnet *netdev = netdev_notifier_info_to_ifp(ptr);
+
+ iwdev = container_of(notifier, struct irdma_device, nb_netdevice_event);
+ if (iwdev->netdev != netdev)
+ return NOTIFY_DONE;
+
+ iwdev->iw_status = 1;
+ switch (event) {
+ case NETDEV_DOWN:
+ iwdev->iw_status = 0;
+ /* fallthrough */
+ case NETDEV_UP:
+ irdma_port_ibevent(iwdev);
+ break;
+ default:
+ break;
+ }
+
+ return NOTIFY_DONE;
+}
+
+void
+irdma_unregister_notifiers(struct irdma_device *iwdev)
+{
+ unregister_netdevice_notifier(&iwdev->nb_netdevice_event);
+}
+
+int
+irdma_register_notifiers(struct irdma_device *iwdev)
+{
+ int ret;
+
+ iwdev->nb_netdevice_event.notifier_call = irdma_netdevice_event;
+ ret = register_netdevice_notifier(&iwdev->nb_netdevice_event);
+ if (ret) {
+ ibdev_err(&iwdev->ibdev, "register_netdevice_notifier failed\n");
+ return ret;
+ }
+ return ret;
+}
/**
* irdma_alloc_and_get_cqp_request - get cqp struct
* @cqp: device cqp ptr
@@ -252,7 +303,7 @@
pending_work = IRDMA_RING_USED_QUANTA(cqp->sc_cqp.sq_ring);
wqe_idx = IRDMA_RING_CURRENT_TAIL(cqp->sc_cqp.sq_ring);
for (i = 0; i < pending_work; i++) {
- cqp_request = (struct irdma_cqp_request *)(unsigned long)
+ cqp_request = (struct irdma_cqp_request *)(uintptr_t)
cqp->scratch_array[wqe_idx];
if (cqp_request)
irdma_free_pending_cqp_request(cqp, cqp_request);
@@ -278,20 +329,23 @@
struct irdma_cqp_request *cqp_request)
{
struct irdma_cqp_timeout cqp_timeout = {0};
+ int timeout_threshold = CQP_TIMEOUT_THRESHOLD;
bool cqp_error = false;
int err_code = 0;
cqp_timeout.compl_cqp_cmds = rf->sc_dev.cqp_cmd_stats[IRDMA_OP_CMPL_CMDS];
do {
+ int wait_time_ms = rf->sc_dev.hw_attrs.max_cqp_compl_wait_time_ms;
+
irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq);
if (wait_event_timeout(cqp_request->waitq,
cqp_request->request_done,
- msecs_to_jiffies(CQP_COMPL_WAIT_TIME_MS)))
+ msecs_to_jiffies(wait_time_ms)))
break;
irdma_check_cqp_progress(&cqp_timeout, &rf->sc_dev);
- if (cqp_timeout.count < CQP_TIMEOUT_THRESHOLD)
+ if (cqp_timeout.count < timeout_threshold)
continue;
if (!rf->reset) {
@@ -304,11 +358,14 @@
cqp_error = cqp_request->compl_info.error;
if (cqp_error) {
err_code = -EIO;
- if (cqp_request->compl_info.maj_err_code == 0xFFFF &&
- cqp_request->compl_info.min_err_code == 0x8029) {
- if (!rf->reset) {
- rf->reset = true;
- rf->gen_ops.request_reset(rf);
+ if (cqp_request->compl_info.maj_err_code == 0xFFFF) {
+ if (cqp_request->compl_info.min_err_code == 0x8002) {
+ err_code = -EBUSY;
+ } else if (cqp_request->compl_info.min_err_code == 0x8029) {
+ if (!rf->reset) {
+ rf->reset = true;
+ rf->gen_ops.request_reset(rf);
+ }
}
}
}
@@ -366,10 +423,12 @@
};
static const struct irdma_cqp_err_info irdma_noncrit_err_list[] = {
+ {0xffff, 0x8002, "Invalid State"},
{0xffff, 0x8006, "Flush No Wqe Pending"},
{0xffff, 0x8007, "Modify QP Bad Close"},
{0xffff, 0x8009, "LLP Closed"},
- {0xffff, 0x800a, "Reset Not Sent"}
+ {0xffff, 0x800a, "Reset Not Sent"},
+ {0xffff, 0x200, "Failover Pending"}
};
/**
@@ -521,16 +580,6 @@
return &iwdev->rf->qp_table[qpn]->ibqp;
}
-/**
- * irdma_get_hw_addr - return hw addr
- * @par: points to shared dev
- */
-u8 __iomem * irdma_get_hw_addr(void *par){
- struct irdma_sc_dev *dev = par;
-
- return dev->hw->hw_addr;
-}
-
/**
* irdma_remove_cqp_head - return head entry and remove
* @dev: device
@@ -713,7 +762,7 @@
*/
int
irdma_cqp_query_fpm_val_cmd(struct irdma_sc_dev *dev,
- struct irdma_dma_mem *val_mem, u8 hmc_fn_id)
+ struct irdma_dma_mem *val_mem, u16 hmc_fn_id)
{
struct irdma_cqp_request *cqp_request;
struct cqp_cmds_info *cqp_info;
@@ -748,7 +797,7 @@
*/
int
irdma_cqp_commit_fpm_val_cmd(struct irdma_sc_dev *dev,
- struct irdma_dma_mem *val_mem, u8 hmc_fn_id)
+ struct irdma_dma_mem *val_mem, u16 hmc_fn_id)
{
struct irdma_cqp_request *cqp_request;
struct cqp_cmds_info *cqp_info;
@@ -1803,7 +1852,7 @@
sizeofbitmap = (u64)pchunk->size >> pprm->pble_shift;
pchunk->bitmapmem.size = sizeofbitmap >> 3;
- pchunk->bitmapmem.va = kzalloc(pchunk->bitmapmem.size, GFP_ATOMIC);
+ pchunk->bitmapmem.va = kzalloc(pchunk->bitmapmem.size, GFP_KERNEL);
if (!pchunk->bitmapmem.va)
return -ENOMEM;
@@ -2056,6 +2105,9 @@
case IRDMA_QP_EVENT_ACCESS_ERR:
ibevent.event = IB_EVENT_QP_ACCESS_ERR;
break;
+ case IRDMA_QP_EVENT_REQ_ERR:
+ ibevent.event = IB_EVENT_QP_REQ_ERR;
+ break;
}
ibevent.device = iwqp->ibqp.device;
ibevent.element.qp = &iwqp->ibqp;
@@ -2168,7 +2220,7 @@
ukcq = &iwcq->sc_cq.cq_uk;
cqe = IRDMA_GET_CURRENT_CQ_ELEM(ukcq);
get_64bit_val(cqe, 24, &qword3);
- polarity = (u8)RS_64(qword3, IRDMA_CQ_VALID);
+ polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3);
return polarity != ukcq->polarity;
}
@@ -2191,7 +2243,7 @@
{
struct irdma_cmpl_gen *cmpl;
- if (!iwcq || list_empty(&iwcq->cmpl_generated))
+ if (list_empty(&iwcq->cmpl_generated))
return -ENOENT;
cmpl = list_first_entry_or_null(&iwcq->cmpl_generated, struct irdma_cmpl_gen, list);
list_del(&cmpl->list);
@@ -2226,7 +2278,10 @@
static inline void
irdma_comp_handler(struct irdma_cq *cq)
{
- if (cq->sc_cq.cq_uk.armed && cq->ibcq.comp_handler)
+ if (!cq->ibcq.comp_handler)
+ return;
+
+ if (atomic_cmpxchg(&cq->armed, 1, 0))
cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
}
@@ -2246,15 +2301,20 @@
u32 wqe_idx;
u8 compl_generated = 0;
unsigned long flags;
+ bool reschedule = false;
#define SQ_COMPL_GENERATED (0x01)
#define RQ_COMPL_GENERATED (0x02)
spin_lock_irqsave(&iwqp->iwscq->lock, flags);
if (irdma_cq_empty(iwqp->iwscq)) {
+ unsigned long flags2;
+
+ spin_lock_irqsave(&iwqp->lock, flags2);
while (IRDMA_RING_MORE_WORK(*sq_ring)) {
- cmpl = kzalloc(sizeof(*cmpl), GFP_KERNEL);
+ cmpl = kzalloc(sizeof(*cmpl), GFP_ATOMIC);
if (!cmpl) {
+ spin_unlock_irqrestore(&iwqp->lock, flags2);
spin_unlock_irqrestore(&iwqp->iwscq->lock, flags);
return;
}
@@ -2266,7 +2326,8 @@
cmpl->cpi.signaled = qp->sq_wrtrk_array[wqe_idx].signaled;
sw_wqe = qp->sq_base[wqe_idx].elem;
get_64bit_val(sw_wqe, IRDMA_BYTE_24, &wqe_qword);
- cmpl->cpi.op_type = (u8)RS_64(wqe_qword, IRDMAQPSQ_OPCODE);
+ cmpl->cpi.op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, wqe_qword);
+ cmpl->cpi.q_type = IRDMA_CQE_QTYPE_SQ;
/* remove the SQ WR by moving SQ tail */
IRDMA_RING_SET_TAIL(*sq_ring, sq_ring->tail + qp->sq_wrtrk_array[sq_ring->tail].quanta);
@@ -2275,16 +2336,22 @@
list_add_tail(&cmpl->list, &iwqp->iwscq->cmpl_generated);
compl_generated |= SQ_COMPL_GENERATED;
}
+ spin_unlock_irqrestore(&iwqp->lock, flags2);
+ spin_unlock_irqrestore(&iwqp->iwscq->lock, flags);
} else {
- mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush, IRDMA_FLUSH_DELAY_MS / 2);
+ spin_unlock_irqrestore(&iwqp->iwscq->lock, flags);
+ reschedule = true;
}
- spin_unlock_irqrestore(&iwqp->iwscq->lock, flags);
spin_lock_irqsave(&iwqp->iwrcq->lock, flags);
if (irdma_cq_empty(iwqp->iwrcq)) {
+ unsigned long flags2;
+
+ spin_lock_irqsave(&iwqp->lock, flags2);
while (IRDMA_RING_MORE_WORK(*rq_ring)) {
- cmpl = kzalloc(sizeof(*cmpl), GFP_KERNEL);
+ cmpl = kzalloc(sizeof(*cmpl), GFP_ATOMIC);
if (!cmpl) {
+ spin_unlock_irqrestore(&iwqp->lock, flags2);
spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags);
return;
}
@@ -2295,6 +2362,7 @@
cmpl->cpi.wr_id = qp->rq_wrid_array[wqe_idx];
cmpl->cpi.signaled = 1;
cmpl->cpi.op_type = IRDMA_OP_TYPE_REC;
+ cmpl->cpi.q_type = IRDMA_CQE_QTYPE_RQ;
/* remove the RQ WR by moving RQ tail */
IRDMA_RING_SET_TAIL(*rq_ring, rq_ring->tail + 1);
irdma_debug(iwqp->sc_qp.dev, IRDMA_DEBUG_DEV,
@@ -2304,22 +2372,26 @@
compl_generated |= RQ_COMPL_GENERATED;
}
+ spin_unlock_irqrestore(&iwqp->lock, flags2);
+ spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags);
} else {
- mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush, IRDMA_FLUSH_DELAY_MS / 2);
+ spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags);
+ reschedule = true;
}
- spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags);
- if (iwqp->iwscq == iwqp->iwrcq) {
- if (compl_generated)
+ if (reschedule)
+ irdma_sched_qp_flush_work(iwqp);
+ if (compl_generated) {
+ if (iwqp->iwscq == iwqp->iwrcq) {
irdma_comp_handler(iwqp->iwscq);
- return;
- }
- if (compl_generated & SQ_COMPL_GENERATED)
- irdma_comp_handler(iwqp->iwscq);
- if (compl_generated & RQ_COMPL_GENERATED)
- irdma_comp_handler(iwqp->iwrcq);
- if (compl_generated)
+ } else {
+ if (compl_generated & SQ_COMPL_GENERATED)
+ irdma_comp_handler(iwqp->iwscq);
+ if (compl_generated & RQ_COMPL_GENERATED)
+ irdma_comp_handler(iwqp->iwrcq);
+ }
irdma_debug(iwqp->sc_qp.dev, IRDMA_DEBUG_VERBS,
"0x%X (SQ 0x1, RQ 0x2, both 0x3) completions generated for QP %d\n",
compl_generated, iwqp->ibqp.qp_num);
+ }
}
diff --git a/sys/dev/irdma/irdma_verbs.h b/sys/dev/irdma/irdma_verbs.h
--- a/sys/dev/irdma/irdma_verbs.h
+++ b/sys/dev/irdma/irdma_verbs.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2021 Intel Corporation
+ * Copyright (c) 2015 - 2022 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -37,7 +37,7 @@
#define IRDMA_VERBS_H
#define IRDMA_MAX_SAVED_PHY_PGADDR 4
-#define IRDMA_FLUSH_DELAY_MS 1500
+#define IRDMA_FLUSH_DELAY_MS 20
#define IRDMA_PKEY_TBL_SZ 1
#define IRDMA_DEFAULT_PKEY 0xFFFF
@@ -132,6 +132,8 @@
struct ib_mw ibmw;
};
struct ib_umem *region;
+ int access;
+ u8 is_hwreg;
u16 type;
u32 page_cnt;
u64 page_size;
@@ -150,7 +152,7 @@
u16 cq_size;
u16 cq_num;
bool user_mode;
- bool armed;
+ atomic_t armed;
enum irdma_cmpl_notify last_notify;
u32 polled_cmpls;
u32 cq_mem_size;
@@ -224,13 +226,6 @@
void *pbl_vbase;
dma_addr_t pbl_pbase;
struct page *page;
- u8 active_conn : 1;
- u8 user_mode : 1;
- u8 hte_added : 1;
- u8 flush_issued : 1;
- u8 sig_all : 1;
- u8 pau_mode : 1;
- u8 rsvd : 1;
u8 iwarp_state;
u16 term_sq_flush_code;
u16 term_rq_flush_code;
@@ -247,6 +242,12 @@
wait_queue_head_t waitq;
wait_queue_head_t mod_qp_waitq;
u8 rts_ae_rcvd;
+ u8 active_conn : 1;
+ u8 user_mode : 1;
+ u8 hte_added : 1;
+ u8 flush_issued : 1;
+ u8 sig_all : 1;
+ u8 pau_mode : 1;
};
enum irdma_mmap_flag {
@@ -262,12 +263,12 @@
static inline u16 irdma_fw_major_ver(struct irdma_sc_dev *dev)
{
- return (u16)RS_64(dev->feature_info[IRDMA_FEATURE_FW_INFO], IRDMA_FW_VER_MAJOR);
+ return (u16)FIELD_GET(IRDMA_FW_VER_MAJOR, dev->feature_info[IRDMA_FEATURE_FW_INFO]);
}
static inline u16 irdma_fw_minor_ver(struct irdma_sc_dev *dev)
{
- return (u16)RS_64(dev->feature_info[IRDMA_FEATURE_FW_INFO], IRDMA_FW_VER_MINOR);
+ return (u16)FIELD_GET(IRDMA_FW_VER_MINOR, dev->feature_info[IRDMA_FEATURE_FW_INFO]);
}
/**
@@ -304,10 +305,10 @@
enum irdma_mmap_flag mmap_flag, u64 *mmap_offset);
int irdma_ib_register_device(struct irdma_device *iwdev);
void irdma_ib_unregister_device(struct irdma_device *iwdev);
-void irdma_ib_dealloc_device(struct ib_device *ibdev);
void irdma_ib_qp_event(struct irdma_qp *iwqp, enum irdma_qp_event_type event);
void irdma_generate_flush_completions(struct irdma_qp *iwqp);
void irdma_remove_cmpls_list(struct irdma_cq *iwcq);
int irdma_generated_cmpls(struct irdma_cq *iwcq, struct irdma_cq_poll_info *cq_poll_info);
+void irdma_sched_qp_flush_work(struct irdma_qp *iwqp);
void irdma_flush_worker(struct work_struct *work);
#endif /* IRDMA_VERBS_H */
diff --git a/sys/dev/irdma/irdma_verbs.c b/sys/dev/irdma/irdma_verbs.c
--- a/sys/dev/irdma/irdma_verbs.c
+++ b/sys/dev/irdma/irdma_verbs.c
@@ -58,33 +58,38 @@
ether_addr_copy((u8 *)&props->sys_image_guid, IF_LLADDR(iwdev->netdev));
props->fw_ver = (u64)irdma_fw_major_ver(&rf->sc_dev) << 32 |
irdma_fw_minor_ver(&rf->sc_dev);
- props->device_cap_flags = iwdev->device_cap_flags;
+ props->device_cap_flags = IB_DEVICE_MEM_WINDOW |
+ IB_DEVICE_MEM_MGT_EXTENSIONS;
+ props->device_cap_flags |= IB_DEVICE_LOCAL_DMA_LKEY;
props->vendor_id = pcidev->vendor;
props->vendor_part_id = pcidev->device;
props->hw_ver = pcidev->revision;
- props->page_size_cap = SZ_4K | SZ_2M | SZ_1G;
+ props->page_size_cap = hw_attrs->page_size_cap;
props->max_mr_size = hw_attrs->max_mr_size;
props->max_qp = rf->max_qp - rf->used_qps;
props->max_qp_wr = hw_attrs->max_qp_wr;
set_max_sge(props, rf);
props->max_cq = rf->max_cq - rf->used_cqs;
- props->max_cqe = rf->max_cqe;
+ props->max_cqe = rf->max_cqe - 1;
props->max_mr = rf->max_mr - rf->used_mrs;
props->max_mw = props->max_mr;
props->max_pd = rf->max_pd - rf->used_pds;
props->max_sge_rd = hw_attrs->uk_attrs.max_hw_read_sges;
props->max_qp_rd_atom = hw_attrs->max_hw_ird;
props->max_qp_init_rd_atom = hw_attrs->max_hw_ord;
- if (rdma_protocol_roce(ibdev, 1))
+ if (rdma_protocol_roce(ibdev, 1)) {
+ props->device_cap_flags |= IB_DEVICE_RC_RNR_NAK_GEN;
props->max_pkeys = IRDMA_PKEY_TBL_SZ;
- props->max_ah = rf->max_ah;
- props->max_mcast_grp = rf->max_mcg;
- props->max_mcast_qp_attach = IRDMA_MAX_MGS_PER_CTX;
- props->max_total_mcast_qp_attach = rf->max_qp * IRDMA_MAX_MGS_PER_CTX;
+ props->max_ah = rf->max_ah;
+ if (hw_attrs->uk_attrs.hw_rev == IRDMA_GEN_2) {
+ props->max_mcast_grp = rf->max_mcg;
+ props->max_mcast_qp_attach = IRDMA_MAX_MGS_PER_CTX;
+ props->max_total_mcast_qp_attach = rf->max_qp * IRDMA_MAX_MGS_PER_CTX;
+ }
+ }
props->max_fast_reg_page_list_len = IRDMA_MAX_PAGES_PER_FMR;
-#define HCA_CLOCK_TIMESTAMP_MASK 0x1ffff
if (hw_attrs->uk_attrs.hw_rev >= IRDMA_GEN_2)
- props->timestamp_mask = HCA_CLOCK_TIMESTAMP_MASK;
+ props->device_cap_flags |= IB_DEVICE_MEM_WINDOW_TYPE_2B;
return 0;
}
@@ -275,6 +280,22 @@
spin_unlock_irqrestore(&iwcq->lock, flags);
}
+static u64 irdma_compute_push_wqe_offset(struct irdma_device *iwdev, u32 page_idx){
+ u64 bar_off = (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET];
+
+ if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_2) {
+ /* skip over db page */
+ bar_off += IRDMA_HW_PAGE_SIZE;
+ /* skip over reserved space */
+ bar_off += IRDMA_PF_BAR_RSVD;
+ }
+
+ /* push wqe page */
+ bar_off += (u64)page_idx * IRDMA_HW_PAGE_SIZE;
+
+ return bar_off;
+}
+
void
irdma_remove_push_mmap_entries(struct irdma_qp *iwqp)
{
@@ -299,17 +320,8 @@
WARN_ON_ONCE(iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev < IRDMA_GEN_2);
- bar_off = (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET];
+ bar_off = irdma_compute_push_wqe_offset(iwdev, iwqp->sc_qp.push_idx);
- if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_2) {
- /* skip over db page */
- bar_off += IRDMA_HW_PAGE_SIZE;
- /* skip over reserved space */
- bar_off += IRDMA_PF_BAR_RSVD;
- }
-
- /* push wqe page */
- bar_off += iwqp->sc_qp.push_idx * IRDMA_HW_PAGE_SIZE;
iwqp->push_wqe_mmap_entry = irdma_user_mmap_entry_insert(ucontext,
bar_off, IRDMA_MMAP_IO_WC,
push_wqe_mmap_key);
@@ -355,6 +367,84 @@
}
}
+/**
+ * irdma_setup_umode_qp - setup sq and rq size in user mode qp
+ * @udata: user data
+ * @iwdev: iwarp device
+ * @iwqp: qp ptr (user or kernel)
+ * @info: initialize info to return
+ * @init_attr: Initial QP create attributes
+ */
+int
+irdma_setup_umode_qp(struct ib_udata *udata,
+ struct irdma_device *iwdev,
+ struct irdma_qp *iwqp,
+ struct irdma_qp_init_info *info,
+ struct ib_qp_init_attr *init_attr)
+{
+ struct irdma_qp_uk_init_info *ukinfo = &info->qp_uk_init_info;
+ struct irdma_create_qp_req req = {0};
+ unsigned long flags;
+ int ret;
+
+ ret = ib_copy_from_udata(&req, udata,
+ min(sizeof(req), udata->inlen));
+ if (ret) {
+ irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS,
+ "ib_copy_from_data fail\n");
+ return ret;
+ }
+
+ iwqp->ctx_info.qp_compl_ctx = req.user_compl_ctx;
+ iwqp->user_mode = 1;
+ if (req.user_wqe_bufs) {
+ struct irdma_ucontext *ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext);
+
+ info->qp_uk_init_info.legacy_mode = ucontext->legacy_mode;
+ spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags);
+ iwqp->iwpbl = irdma_get_pbl((unsigned long)req.user_wqe_bufs,
+ &ucontext->qp_reg_mem_list);
+ spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags);
+
+ if (!iwqp->iwpbl) {
+ ret = -ENODATA;
+ irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS,
+ "no pbl info\n");
+ return ret;
+ }
+ }
+
+ if (ukinfo->abi_ver <= 5) {
+ /**
+ * For ABI version less than 6 passes raw sq and rq
+ * quanta in cap.max_send_wr and cap.max_recv_wr.
+ */
+ iwqp->max_send_wr = init_attr->cap.max_send_wr;
+ iwqp->max_recv_wr = init_attr->cap.max_recv_wr;
+ ukinfo->sq_size = init_attr->cap.max_send_wr;
+ ukinfo->rq_size = init_attr->cap.max_recv_wr;
+ irdma_uk_calc_shift_wq(ukinfo, &ukinfo->sq_shift, &ukinfo->rq_shift);
+ } else {
+ ret = irdma_uk_calc_depth_shift_sq(ukinfo, &ukinfo->sq_depth,
+ &ukinfo->sq_shift);
+ if (ret)
+ return ret;
+
+ ret = irdma_uk_calc_depth_shift_rq(ukinfo, &ukinfo->rq_depth,
+ &ukinfo->rq_shift);
+ if (ret)
+ return ret;
+
+ iwqp->max_send_wr = (ukinfo->sq_depth - IRDMA_SQ_RSVD) >> ukinfo->sq_shift;
+ iwqp->max_recv_wr = (ukinfo->rq_depth - IRDMA_RQ_RSVD) >> ukinfo->rq_shift;
+ ukinfo->sq_size = ukinfo->sq_depth >> ukinfo->sq_shift;
+ ukinfo->rq_size = ukinfo->rq_depth >> ukinfo->rq_shift;
+ }
+ irdma_setup_virt_qp(iwdev, iwqp, info);
+
+ return 0;
+}
+
/**
* irdma_setup_kmode_qp - setup initialization for kernel mode qp
* @iwdev: iwarp device
@@ -369,48 +459,35 @@
struct ib_qp_init_attr *init_attr)
{
struct irdma_dma_mem *mem = &iwqp->kqp.dma_mem;
- u32 sqdepth, rqdepth;
- u8 sqshift, rqshift;
u32 size;
int status;
struct irdma_qp_uk_init_info *ukinfo = &info->qp_uk_init_info;
- struct irdma_uk_attrs *uk_attrs = &iwdev->rf->sc_dev.hw_attrs.uk_attrs;
-
- irdma_get_wqe_shift(uk_attrs,
- uk_attrs->hw_rev >= IRDMA_GEN_2 ? ukinfo->max_sq_frag_cnt + 1 :
- ukinfo->max_sq_frag_cnt,
- ukinfo->max_inline_data, &sqshift);
- status = irdma_get_sqdepth(uk_attrs->max_hw_wq_quanta, ukinfo->sq_size,
- sqshift, &sqdepth);
+
+ status = irdma_uk_calc_depth_shift_sq(ukinfo, &ukinfo->sq_depth,
+ &ukinfo->sq_shift);
if (status)
return status;
- if (uk_attrs->hw_rev == IRDMA_GEN_1)
- rqshift = IRDMA_MAX_RQ_WQE_SHIFT_GEN1;
- else
- irdma_get_wqe_shift(uk_attrs, ukinfo->max_rq_frag_cnt, 0,
- &rqshift);
-
- status = irdma_get_rqdepth(uk_attrs->max_hw_rq_quanta, ukinfo->rq_size,
- rqshift, &rqdepth);
+ status = irdma_uk_calc_depth_shift_rq(ukinfo, &ukinfo->rq_depth,
+ &ukinfo->rq_shift);
if (status)
return status;
iwqp->kqp.sq_wrid_mem =
- kcalloc(sqdepth, sizeof(*iwqp->kqp.sq_wrid_mem), GFP_KERNEL);
+ kcalloc(ukinfo->sq_depth, sizeof(*iwqp->kqp.sq_wrid_mem), GFP_KERNEL);
if (!iwqp->kqp.sq_wrid_mem)
return -ENOMEM;
iwqp->kqp.rq_wrid_mem =
- kcalloc(rqdepth, sizeof(*iwqp->kqp.rq_wrid_mem), GFP_KERNEL);
+ kcalloc(ukinfo->rq_depth, sizeof(*iwqp->kqp.rq_wrid_mem), GFP_KERNEL);
if (!iwqp->kqp.rq_wrid_mem) {
kfree(iwqp->kqp.sq_wrid_mem);
iwqp->kqp.sq_wrid_mem = NULL;
return -ENOMEM;
}
- iwqp->kqp.sig_trk_mem = kcalloc(sqdepth, sizeof(u32), GFP_KERNEL);
- memset(iwqp->kqp.sig_trk_mem, 0, sqdepth * sizeof(u32));
+ iwqp->kqp.sig_trk_mem = kcalloc(ukinfo->sq_depth, sizeof(u32), GFP_KERNEL);
+ memset(iwqp->kqp.sig_trk_mem, 0, ukinfo->sq_depth * sizeof(u32));
if (!iwqp->kqp.sig_trk_mem) {
kfree(iwqp->kqp.sq_wrid_mem);
iwqp->kqp.sq_wrid_mem = NULL;
@@ -422,7 +499,7 @@
ukinfo->sq_wrtrk_array = iwqp->kqp.sq_wrid_mem;
ukinfo->rq_wrid_array = iwqp->kqp.rq_wrid_mem;
- size = (sqdepth + rqdepth) * IRDMA_QP_WQE_MIN_SIZE;
+ size = (ukinfo->sq_depth + ukinfo->rq_depth) * IRDMA_QP_WQE_MIN_SIZE;
size += (IRDMA_SHADOW_AREA_SIZE << 3);
mem->size = size;
@@ -438,16 +515,18 @@
ukinfo->sq = mem->va;
info->sq_pa = mem->pa;
- ukinfo->rq = &ukinfo->sq[sqdepth];
- info->rq_pa = info->sq_pa + (sqdepth * IRDMA_QP_WQE_MIN_SIZE);
- ukinfo->shadow_area = ukinfo->rq[rqdepth].elem;
- info->shadow_area_pa = info->rq_pa + (rqdepth * IRDMA_QP_WQE_MIN_SIZE);
- ukinfo->sq_size = sqdepth >> sqshift;
- ukinfo->rq_size = rqdepth >> rqshift;
+ ukinfo->rq = &ukinfo->sq[ukinfo->sq_depth];
+ info->rq_pa = info->sq_pa + (ukinfo->sq_depth * IRDMA_QP_WQE_MIN_SIZE);
+ ukinfo->shadow_area = ukinfo->rq[ukinfo->rq_depth].elem;
+ info->shadow_area_pa = info->rq_pa + (ukinfo->rq_depth * IRDMA_QP_WQE_MIN_SIZE);
+ ukinfo->sq_size = ukinfo->sq_depth >> ukinfo->sq_shift;
+ ukinfo->rq_size = ukinfo->rq_depth >> ukinfo->rq_shift;
ukinfo->qp_id = iwqp->ibqp.qp_num;
- init_attr->cap.max_send_wr = (sqdepth - IRDMA_SQ_RSVD) >> sqshift;
- init_attr->cap.max_recv_wr = (rqdepth - IRDMA_RQ_RSVD) >> rqshift;
+ iwqp->max_send_wr = (ukinfo->sq_depth - IRDMA_SQ_RSVD) >> ukinfo->sq_shift;
+ iwqp->max_recv_wr = (ukinfo->rq_depth - IRDMA_RQ_RSVD) >> ukinfo->rq_shift;
+ init_attr->cap.max_send_wr = iwqp->max_send_wr;
+ init_attr->cap.max_recv_wr = iwqp->max_recv_wr;
return 0;
}
@@ -582,16 +661,24 @@
return 0;
}
+void
+irdma_sched_qp_flush_work(struct irdma_qp *iwqp)
+{
+ irdma_qp_add_ref(&iwqp->ibqp);
+ if (mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush,
+ msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS)))
+ irdma_qp_rem_ref(&iwqp->ibqp);
+}
+
void
irdma_flush_worker(struct work_struct *work)
{
struct delayed_work *dwork = to_delayed_work(work);
struct irdma_qp *iwqp = container_of(dwork, struct irdma_qp, dwork_flush);
- unsigned long flags;
- spin_lock_irqsave(&iwqp->lock, flags); /* Don't allow more posting while generating completions */
irdma_generate_flush_completions(iwqp);
- spin_unlock_irqrestore(&iwqp->lock, flags);
+ /* For the add in irdma_sched_qp_flush_work */
+ irdma_qp_rem_ref(&iwqp->ibqp);
}
static int
@@ -680,6 +767,8 @@
irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr,
int attr_mask, struct ib_udata *udata)
{
+#define IRDMA_MODIFY_QP_MIN_REQ_LEN offsetofend(struct irdma_modify_qp_req, rq_flush)
+#define IRDMA_MODIFY_QP_MIN_RESP_LEN offsetofend(struct irdma_modify_qp_resp, push_valid)
struct irdma_pd *iwpd = to_iwpd(ibqp->pd);
struct irdma_qp *iwqp = to_iwqp(ibqp);
struct irdma_device *iwdev = iwqp->iwdev;
@@ -689,7 +778,7 @@
struct irdma_udp_offload_info *udp_info;
struct irdma_modify_qp_info info = {0};
struct irdma_modify_qp_resp uresp = {};
- struct irdma_modify_qp_req ureq = {};
+ struct irdma_modify_qp_req ureq;
unsigned long flags;
u8 issue_modify_qp = 0;
int ret = 0;
@@ -698,6 +787,12 @@
roce_info = &iwqp->roce_info;
udp_info = &iwqp->udp_info;
+ if (udata) {
+ if ((udata->inlen && udata->inlen < IRDMA_MODIFY_QP_MIN_REQ_LEN) ||
+ (udata->outlen && udata->outlen < IRDMA_MODIFY_QP_MIN_RESP_LEN))
+ return -EINVAL;
+ }
+
if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
return -EOPNOTSUPP;
@@ -745,6 +840,11 @@
udp_info->ttl = attr->ah_attr.grh.hop_limit;
udp_info->flow_label = attr->ah_attr.grh.flow_label;
udp_info->tos = attr->ah_attr.grh.traffic_class;
+
+ udp_info->src_port = kc_rdma_get_udp_sport(udp_info->flow_label,
+ ibqp->qp_num,
+ roce_info->dest_qp);
+
irdma_qp_rem_qos(&iwqp->sc_qp);
dev->ws_remove(iwqp->sc_qp.vsi, ctx_info->user_pri);
if (iwqp->sc_qp.vsi->dscp_mode)
@@ -752,14 +852,14 @@
iwqp->sc_qp.vsi->dscp_map[irdma_tos2dscp(udp_info->tos)];
else
ctx_info->user_pri = rt_tos2priority(udp_info->tos);
- iwqp->sc_qp.user_pri = ctx_info->user_pri;
- if (dev->ws_add(iwqp->sc_qp.vsi, ctx_info->user_pri))
- return -ENOMEM;
- irdma_qp_add_qos(&iwqp->sc_qp);
}
ret = kc_irdma_set_roce_cm_info(iwqp, attr, &vlan_id);
if (ret)
return ret;
+ if (dev->ws_add(iwqp->sc_qp.vsi, ctx_info->user_pri))
+ return -ENOMEM;
+ iwqp->sc_qp.user_pri = ctx_info->user_pri;
+ irdma_qp_add_qos(&iwqp->sc_qp);
if (vlan_id >= VLAN_N_VID && iwdev->dcb_vlan_mode)
vlan_id = 0;
@@ -773,7 +873,6 @@
av->attrs = attr->ah_attr;
rdma_gid2ip((struct sockaddr *)&av->dgid_addr, &attr->ah_attr.grh.dgid);
- roce_info->local_qp = ibqp->qp_num;
if (av->sgid_addr.saddr.sa_family == AF_INET6) {
__be32 *daddr =
av->dgid_addr.saddr_in6.sin6_addr.__u6_addr.__u6_addr32;
@@ -853,7 +952,9 @@
spin_lock_irqsave(&iwqp->lock, flags);
if (attr_mask & IB_QP_STATE) {
- if (!ib_modify_qp_is_ok(iwqp->ibqp_state, attr->qp_state, iwqp->ibqp.qp_type, attr_mask)) {
+ if (!kc_ib_modify_qp_is_ok(iwqp->ibqp_state, attr->qp_state,
+ iwqp->ibqp.qp_type, attr_mask,
+ IB_LINK_LAYER_ETHERNET)) {
irdma_print("modify_qp invalid for qp_id=%d, old_state=0x%x, new_state=0x%x\n",
iwqp->ibqp.qp_num, iwqp->ibqp_state,
attr->qp_state);
@@ -932,7 +1033,7 @@
if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) {
spin_unlock_irqrestore(&iwqp->lock, flags);
- if (udata) {
+ if (udata && udata->inlen) {
if (ib_copy_from_udata(&ureq, udata,
min(sizeof(ureq), udata->inlen)))
return -EINVAL;
@@ -973,22 +1074,19 @@
}
if (iwqp->ibqp_state > IB_QPS_RTS &&
!iwqp->flush_issued) {
- iwqp->flush_issued = 1;
- if (!iwqp->user_mode)
- queue_delayed_work(iwqp->iwdev->cleanup_wq,
- &iwqp->dwork_flush,
- msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS));
spin_unlock_irqrestore(&iwqp->lock, flags);
irdma_flush_wqes(iwqp, IRDMA_FLUSH_SQ |
IRDMA_FLUSH_RQ |
IRDMA_FLUSH_WAIT);
+ iwqp->flush_issued = 1;
+
} else {
spin_unlock_irqrestore(&iwqp->lock, flags);
}
} else {
iwqp->ibqp_state = attr->qp_state;
}
- if (udata && dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
+ if (udata && udata->outlen && dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
struct irdma_ucontext *ucontext;
ucontext = rdma_udata_to_drv_context(udata,
@@ -1031,6 +1129,8 @@
irdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
struct ib_udata *udata)
{
+#define IRDMA_MODIFY_QP_MIN_REQ_LEN offsetofend(struct irdma_modify_qp_req, rq_flush)
+#define IRDMA_MODIFY_QP_MIN_RESP_LEN offsetofend(struct irdma_modify_qp_resp, push_valid)
struct irdma_qp *iwqp = to_iwqp(ibqp);
struct irdma_device *iwdev = iwqp->iwdev;
struct irdma_sc_dev *dev = &iwdev->rf->sc_dev;
@@ -1045,6 +1145,12 @@
int err;
unsigned long flags;
+ if (udata) {
+ if ((udata->inlen && udata->inlen < IRDMA_MODIFY_QP_MIN_REQ_LEN) ||
+ (udata->outlen && udata->outlen < IRDMA_MODIFY_QP_MIN_RESP_LEN))
+ return -EINVAL;
+ }
+
if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
return -EOPNOTSUPP;
@@ -1122,12 +1228,14 @@
goto exit;
}
- /* fallthrough */
+ info.next_iwarp_state = IRDMA_QP_STATE_TERMINATE;
+ issue_modify_qp = 1;
+ break;
case IB_QPS_ERR:
case IB_QPS_RESET:
if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) {
spin_unlock_irqrestore(&iwqp->lock, flags);
- if (udata) {
+ if (udata && udata->inlen) {
if (ib_copy_from_udata(&ureq, udata,
min(sizeof(ureq), udata->inlen)))
return -EINVAL;
@@ -1197,13 +1305,13 @@
if (issue_modify_qp && iwqp->ibqp_state > IB_QPS_RTS) {
if (dont_wait) {
- if (iwqp->cm_id && iwqp->hw_tcp_state) {
+ if (iwqp->hw_tcp_state) {
spin_lock_irqsave(&iwqp->lock, flags);
iwqp->hw_tcp_state = IRDMA_TCP_STATE_CLOSED;
iwqp->last_aeq = IRDMA_AE_RESET_SENT;
spin_unlock_irqrestore(&iwqp->lock, flags);
- irdma_cm_disconn(iwqp);
}
+ irdma_cm_disconn(iwqp);
} else {
int close_timer_started;
@@ -1224,7 +1332,7 @@
}
}
}
- if (attr_mask & IB_QP_STATE && udata &&
+ if (attr_mask & IB_QP_STATE && udata && udata->outlen &&
dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
struct irdma_ucontext *ucontext;
@@ -1325,6 +1433,7 @@
irdma_resize_cq(struct ib_cq *ibcq, int entries,
struct ib_udata *udata)
{
+#define IRDMA_RESIZE_CQ_MIN_REQ_LEN offsetofend(struct irdma_resize_cq_req, user_cq_buffer)
struct irdma_cq *iwcq = to_iwcq(ibcq);
struct irdma_sc_dev *dev = iwcq->sc_cq.dev;
struct irdma_cqp_request *cqp_request;
@@ -1347,6 +1456,9 @@
IRDMA_FEATURE_CQ_RESIZE))
return -EOPNOTSUPP;
+ if (udata && udata->inlen < IRDMA_RESIZE_CQ_MIN_REQ_LEN)
+ return -EINVAL;
+
if (entries > rf->max_cqe)
return -EINVAL;
@@ -1362,7 +1474,7 @@
return 0;
if (udata) {
- struct irdma_resize_cq_req req = {0};
+ struct irdma_resize_cq_req req = {};
struct irdma_ucontext *ucontext =
rdma_udata_to_drv_context(udata, struct irdma_ucontext,
ibucontext);
@@ -1499,7 +1611,7 @@
u32
irdma_create_stag(struct irdma_device *iwdev)
{
- u32 stag = 0;
+ u32 stag;
u32 stag_index = 0;
u32 next_stag_index;
u32 driver_key;
@@ -1518,7 +1630,7 @@
iwdev->rf->max_mr, &stag_index,
&next_stag_index);
if (ret)
- return stag;
+ return 0;
stag = stag_index << IRDMA_CQPSQ_STAG_IDX_S;
stag |= driver_key;
stag += (u32)consumer_key;
@@ -1588,10 +1700,11 @@
* @rf: RDMA PCI function
* @iwmr: mr pointer for this memory registration
* @use_pbles: flag if to use pble's
+ * @lvl_1_only: request only level 1 pble if true
*/
static int
irdma_setup_pbles(struct irdma_pci_f *rf, struct irdma_mr *iwmr,
- bool use_pbles)
+ bool use_pbles, bool lvl_1_only)
{
struct irdma_pbl *iwpbl = &iwmr->iwpbl;
struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc;
@@ -1602,7 +1715,7 @@
if (use_pbles) {
status = irdma_get_pble(rf->pble_rsrc, palloc, iwmr->page_cnt,
- false);
+ lvl_1_only);
if (status)
return status;
@@ -1646,16 +1759,10 @@
bool ret = true;
pg_size = iwmr->page_size;
- err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles);
+ err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles, true);
if (err)
return err;
- if (use_pbles && palloc->level != PBLE_LEVEL_1) {
- irdma_free_pble(iwdev->rf->pble_rsrc, palloc);
- iwpbl->pbl_allocated = false;
- return -ENOMEM;
- }
-
if (use_pbles)
arr = palloc->level1.addr;
@@ -1772,7 +1879,7 @@
cqp_info = &cqp_request->info;
info = &cqp_info->in.u.dealloc_stag.info;
memset(info, 0, sizeof(*info));
- info->pd_id = iwpd->sc_pd.pd_id & 0x00007fff;
+ info->pd_id = iwpd->sc_pd.pd_id;
info->stag_idx = RS_64_1(ibmw->rkey, IRDMA_CQPSQ_STAG_IDX_S);
info->mr = false;
cqp_info->cqp_cmd = IRDMA_OP_DEALLOC_STAG;
@@ -1797,10 +1904,11 @@
struct irdma_mr *iwmr)
{
struct irdma_allocate_stag_info *info;
- struct irdma_pd *iwpd = to_iwpd(iwmr->ibmr.pd);
- int status;
+ struct ib_pd *pd = iwmr->ibmr.pd;
+ struct irdma_pd *iwpd = to_iwpd(pd);
struct irdma_cqp_request *cqp_request;
struct cqp_cmds_info *cqp_info;
+ int status;
cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true);
if (!cqp_request)
@@ -1813,6 +1921,7 @@
info->stag_idx = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S;
info->pd_id = iwpd->sc_pd.pd_id;
info->total_len = iwmr->len;
+ info->all_memory = (pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY) ? true : false;
info->remote_access = true;
cqp_info->cqp_cmd = IRDMA_OP_ALLOC_STAG;
cqp_info->post_sq = 1;
@@ -1820,6 +1929,8 @@
cqp_info->in.u.alloc_stag.scratch = (uintptr_t)cqp_request;
status = irdma_handle_cqp_op(iwdev->rf, cqp_request);
irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request);
+ if (!status)
+ iwmr->is_hwreg = 1;
return status;
}
@@ -1840,9 +1951,17 @@
if (unlikely(iwmr->npages == iwmr->page_cnt))
return -ENOMEM;
- pbl = palloc->level1.addr;
- pbl[iwmr->npages++] = addr;
+ if (palloc->level == PBLE_LEVEL_2) {
+ struct irdma_pble_info *palloc_info =
+ palloc->level2.leaf + (iwmr->npages >> PBLE_512_SHIFT);
+
+ palloc_info->addr[iwmr->npages & (PBLE_PER_PAGE - 1)] = addr;
+ } else {
+ pbl = palloc->level1.addr;
+ pbl[iwmr->npages] = addr;
+ }
+ iwmr->npages++;
return 0;
}
@@ -1870,13 +1989,14 @@
* @iwmr: irdma mr pointer
* @access: access for MR
*/
-static int
+int
irdma_hwreg_mr(struct irdma_device *iwdev, struct irdma_mr *iwmr,
u16 access)
{
struct irdma_pbl *iwpbl = &iwmr->iwpbl;
struct irdma_reg_ns_stag_info *stag_info;
- struct irdma_pd *iwpd = to_iwpd(iwmr->ibmr.pd);
+ struct ib_pd *pd = iwmr->ibmr.pd;
+ struct irdma_pd *iwpd = to_iwpd(pd);
struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc;
struct irdma_cqp_request *cqp_request;
struct cqp_cmds_info *cqp_info;
@@ -1893,6 +2013,7 @@
stag_info->stag_idx = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S;
stag_info->stag_key = (u8)iwmr->stag;
stag_info->total_len = iwmr->len;
+ stag_info->all_memory = (pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY) ? true : false;
stag_info->access_rights = irdma_get_mr_access(access);
stag_info->pd_id = iwpd->sc_pd.pd_id;
if (stag_info->access_rights & IRDMA_ACCESS_FLAGS_ZERO_BASED)
@@ -1920,6 +2041,9 @@
ret = irdma_handle_cqp_op(iwdev->rf, cqp_request);
irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request);
+ if (!ret)
+ iwmr->is_hwreg = 1;
+
return ret;
}
@@ -1937,13 +2061,14 @@
u64 virt, int access,
struct ib_udata *udata)
{
+#define IRDMA_MEM_REG_MIN_REQ_LEN offsetofend(struct irdma_mem_reg_req, sq_pages)
struct irdma_device *iwdev = to_iwdev(pd->device);
struct irdma_ucontext *ucontext;
struct irdma_pble_alloc *palloc;
struct irdma_pbl *iwpbl;
struct irdma_mr *iwmr;
struct ib_umem *region;
- struct irdma_mem_reg_req req;
+ struct irdma_mem_reg_req req = {};
u32 total, stag = 0;
u8 shadow_pgcnt = 1;
bool use_pbles = false;
@@ -1951,7 +2076,10 @@
int err = -EINVAL;
int ret;
- if (!len || len > iwdev->rf->sc_dev.hw_attrs.max_mr_size)
+ if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size)
+ return ERR_PTR(-EINVAL);
+
+ if (udata->inlen < IRDMA_MEM_REG_MIN_REQ_LEN)
return ERR_PTR(-EINVAL);
region = ib_umem_get(pd->uobject->context, start, len, access, 0);
@@ -1979,9 +2107,9 @@
iwmr->ibmr.pd = pd;
iwmr->ibmr.device = pd->device;
iwmr->ibmr.iova = virt;
- iwmr->page_size = PAGE_SIZE;
+ iwmr->page_size = IRDMA_HW_PAGE_SIZE;
+ iwmr->page_msk = ~(IRDMA_HW_PAGE_SIZE - 1);
- iwmr->page_msk = PAGE_MASK;
iwmr->len = region->length;
iwpbl->user_base = virt;
palloc = &iwpbl->pble_alloc;
@@ -2032,7 +2160,7 @@
case IRDMA_MEMREG_TYPE_MEM:
use_pbles = (iwmr->page_cnt != 1);
- err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles);
+ err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles, false);
if (err)
goto error;
@@ -2054,6 +2182,7 @@
iwmr->stag = stag;
iwmr->ibmr.rkey = stag;
iwmr->ibmr.lkey = stag;
+ iwmr->access = access;
err = irdma_hwreg_mr(iwdev, iwmr, access);
if (err) {
irdma_free_stag(iwdev, stag);
@@ -2078,6 +2207,120 @@
return ERR_PTR(err);
}
+int
+irdma_hwdereg_mr(struct ib_mr *ib_mr)
+{
+ struct irdma_device *iwdev = to_iwdev(ib_mr->device);
+ struct irdma_mr *iwmr = to_iwmr(ib_mr);
+ struct irdma_pd *iwpd = to_iwpd(ib_mr->pd);
+ struct irdma_dealloc_stag_info *info;
+ struct irdma_pbl *iwpbl = &iwmr->iwpbl;
+ struct irdma_cqp_request *cqp_request;
+ struct cqp_cmds_info *cqp_info;
+ int status;
+
+ /*
+ * Skip HW MR de-register when it is already de-registered during an MR re-reregister and the re-registration
+ * fails
+ */
+ if (!iwmr->is_hwreg)
+ return 0;
+
+ cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true);
+ if (!cqp_request)
+ return -ENOMEM;
+
+ cqp_info = &cqp_request->info;
+ info = &cqp_info->in.u.dealloc_stag.info;
+ memset(info, 0, sizeof(*info));
+ info->pd_id = iwpd->sc_pd.pd_id;
+ info->stag_idx = RS_64_1(ib_mr->rkey, IRDMA_CQPSQ_STAG_IDX_S);
+ info->mr = true;
+ if (iwpbl->pbl_allocated)
+ info->dealloc_pbl = true;
+
+ cqp_info->cqp_cmd = IRDMA_OP_DEALLOC_STAG;
+ cqp_info->post_sq = 1;
+ cqp_info->in.u.dealloc_stag.dev = &iwdev->rf->sc_dev;
+ cqp_info->in.u.dealloc_stag.scratch = (uintptr_t)cqp_request;
+ status = irdma_handle_cqp_op(iwdev->rf, cqp_request);
+ irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request);
+
+ if (!status)
+ iwmr->is_hwreg = 0;
+
+ return status;
+}
+
+/*
+ * irdma_rereg_mr_trans - Re-register a user MR for a change translation. @iwmr: ptr of iwmr @start: virtual start
+ * address @len: length of mr @virt: virtual address
+ *
+ * Re-register a user memory region when a change translation is requested. Re-register a new region while reusing the
+ * stag from the original registration.
+ */
+struct ib_mr *
+irdma_rereg_mr_trans(struct irdma_mr *iwmr, u64 start, u64 len,
+ u64 virt, struct ib_udata *udata)
+{
+ struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device);
+ struct irdma_pbl *iwpbl = &iwmr->iwpbl;
+ struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc;
+ struct ib_pd *pd = iwmr->ibmr.pd;
+ struct ib_umem *region;
+ bool use_pbles;
+ int err;
+
+ region = ib_umem_get(pd->uobject->context, start, len, iwmr->access, 0);
+
+ if (IS_ERR(region)) {
+ irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS,
+ "Failed to create ib_umem region\n");
+ return (struct ib_mr *)region;
+ }
+
+ iwmr->region = region;
+ iwmr->ibmr.iova = virt;
+ iwmr->ibmr.pd = pd;
+ iwmr->page_size = PAGE_SIZE;
+
+ iwmr->len = region->length;
+ iwpbl->user_base = virt;
+ iwmr->page_cnt = irdma_ib_umem_num_dma_blocks(region, iwmr->page_size,
+ virt);
+
+ use_pbles = (iwmr->page_cnt != 1);
+
+ err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles, false);
+ if (err)
+ goto error;
+
+ if (use_pbles) {
+ err = irdma_check_mr_contiguous(palloc,
+ iwmr->page_size);
+ if (err) {
+ irdma_free_pble(iwdev->rf->pble_rsrc, palloc);
+ iwpbl->pbl_allocated = false;
+ }
+ }
+
+ err = irdma_hwreg_mr(iwdev, iwmr, iwmr->access);
+ if (err)
+ goto error;
+
+ return &iwmr->ibmr;
+
+error:
+ if (palloc->level != PBLE_LEVEL_0 && iwpbl->pbl_allocated) {
+ irdma_free_pble(iwdev->rf->pble_rsrc, palloc);
+ iwpbl->pbl_allocated = false;
+ }
+ ib_umem_release(region);
+ iwmr->region = NULL;
+
+ return ERR_PTR(err);
+}
+
/**
* irdma_reg_phys_mr - register kernel physical memory
* @pd: ibpd pointer
@@ -2259,31 +2502,20 @@
info.stag_to_inv = ib_wr->ex.invalidate_rkey;
}
- if (ib_wr->send_flags & IB_SEND_INLINE) {
- info.op.inline_send.data = (void *)(unsigned long)
- ib_wr->sg_list[0].addr;
- info.op.inline_send.len = ib_wr->sg_list[0].length;
- if (iwqp->ibqp.qp_type == IB_QPT_UD ||
- iwqp->ibqp.qp_type == IB_QPT_GSI) {
- ah = to_iwah(ud_wr(ib_wr)->ah);
- info.op.inline_send.ah_id = ah->sc_ah.ah_info.ah_idx;
- info.op.inline_send.qkey = ud_wr(ib_wr)->remote_qkey;
- info.op.inline_send.dest_qp = ud_wr(ib_wr)->remote_qpn;
- }
+ info.op.send.num_sges = ib_wr->num_sge;
+ info.op.send.sg_list = (struct irdma_sge *)ib_wr->sg_list;
+ if (iwqp->ibqp.qp_type == IB_QPT_UD ||
+ iwqp->ibqp.qp_type == IB_QPT_GSI) {
+ ah = to_iwah(ud_wr(ib_wr)->ah);
+ info.op.send.ah_id = ah->sc_ah.ah_info.ah_idx;
+ info.op.send.qkey = ud_wr(ib_wr)->remote_qkey;
+ info.op.send.dest_qp = ud_wr(ib_wr)->remote_qpn;
+ }
+
+ if (ib_wr->send_flags & IB_SEND_INLINE)
err = irdma_uk_inline_send(ukqp, &info, false);
- } else {
- info.op.send.num_sges = ib_wr->num_sge;
- info.op.send.sg_list = (struct irdma_sge *)
- ib_wr->sg_list;
- if (iwqp->ibqp.qp_type == IB_QPT_UD ||
- iwqp->ibqp.qp_type == IB_QPT_GSI) {
- ah = to_iwah(ud_wr(ib_wr)->ah);
- info.op.send.ah_id = ah->sc_ah.ah_info.ah_idx;
- info.op.send.qkey = ud_wr(ib_wr)->remote_qkey;
- info.op.send.dest_qp = ud_wr(ib_wr)->remote_qpn;
- }
+ else
err = irdma_uk_send(ukqp, &info, false);
- }
break;
case IB_WR_RDMA_WRITE_WITH_IMM:
if (ukqp->qp_caps & IRDMA_WRITE_WITH_IMM) {
@@ -2300,19 +2532,14 @@
else
info.op_type = IRDMA_OP_TYPE_RDMA_WRITE;
- if (ib_wr->send_flags & IB_SEND_INLINE) {
- info.op.inline_rdma_write.data = (void *)(uintptr_t)ib_wr->sg_list[0].addr;
- info.op.inline_rdma_write.len = ib_wr->sg_list[0].length;
- info.op.inline_rdma_write.rem_addr.tag_off = rdma_wr(ib_wr)->remote_addr;
- info.op.inline_rdma_write.rem_addr.stag = rdma_wr(ib_wr)->rkey;
+ info.op.rdma_write.num_lo_sges = ib_wr->num_sge;
+ info.op.rdma_write.lo_sg_list = (void *)ib_wr->sg_list;
+ info.op.rdma_write.rem_addr.tag_off = rdma_wr(ib_wr)->remote_addr;
+ info.op.rdma_write.rem_addr.stag = rdma_wr(ib_wr)->rkey;
+ if (ib_wr->send_flags & IB_SEND_INLINE)
err = irdma_uk_inline_rdma_write(ukqp, &info, false);
- } else {
- info.op.rdma_write.lo_sg_list = (void *)ib_wr->sg_list;
- info.op.rdma_write.num_lo_sges = ib_wr->num_sge;
- info.op.rdma_write.rem_addr.tag_off = rdma_wr(ib_wr)->remote_addr;
- info.op.rdma_write.rem_addr.stag = rdma_wr(ib_wr)->rkey;
+ else
err = irdma_uk_rdma_write(ukqp, &info, false);
- }
break;
case IB_WR_RDMA_READ_WITH_INV:
inv_stag = true;
@@ -2350,11 +2577,14 @@
stag_info.addr_type = IRDMA_ADDR_TYPE_VA_BASED;
stag_info.va = (void *)(uintptr_t)iwmr->ibmr.iova;
stag_info.total_len = iwmr->ibmr.length;
- stag_info.reg_addr_pa = *palloc->level1.addr;
- stag_info.first_pm_pbl_index = palloc->level1.idx;
- stag_info.local_fence = ib_wr->send_flags & IB_SEND_FENCE;
- if (iwmr->npages > IRDMA_MIN_PAGES_PER_FMR)
+ if (palloc->level == PBLE_LEVEL_2) {
+ stag_info.chunk_size = 3;
+ stag_info.first_pm_pbl_index = palloc->level2.root.idx;
+ } else {
stag_info.chunk_size = 1;
+ stag_info.first_pm_pbl_index = palloc->level1.idx;
+ }
+ stag_info.local_fence = ib_wr->send_flags & IB_SEND_FENCE;
err = irdma_sc_mr_fast_register(&iwqp->sc_qp, &stag_info,
true);
break;
@@ -2373,11 +2603,14 @@
ib_wr = ib_wr->next;
}
- if (!iwqp->flush_issued && iwqp->hw_iwarp_state <= IRDMA_QP_STATE_RTS)
- irdma_uk_qp_post_wr(ukqp);
- else if (iwqp->flush_issued)
- mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush, IRDMA_FLUSH_DELAY_MS);
- spin_unlock_irqrestore(&iwqp->lock, flags);
+ if (!iwqp->flush_issued) {
+ if (iwqp->hw_iwarp_state <= IRDMA_QP_STATE_RTS)
+ irdma_uk_qp_post_wr(ukqp);
+ spin_unlock_irqrestore(&iwqp->lock, flags);
+ } else {
+ spin_unlock_irqrestore(&iwqp->lock, flags);
+ irdma_sched_qp_flush_work(iwqp);
+ }
if (err)
*bad_wr = ib_wr;
@@ -2425,9 +2658,9 @@
}
out:
- if (iwqp->flush_issued)
- mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush, IRDMA_FLUSH_DELAY_MS);
spin_unlock_irqrestore(&iwqp->lock, flags);
+ if (iwqp->flush_issued)
+ irdma_sched_qp_flush_work(iwqp);
if (err)
*bad_wr = ib_wr;
@@ -2456,6 +2689,8 @@
return IB_WC_WR_FLUSH_ERR;
case FLUSH_MW_BIND_ERR:
return IB_WC_MW_BIND_ERR;
+ case FLUSH_REM_INV_REQ_ERR:
+ return IB_WC_REM_INV_REQ_ERR;
case FLUSH_RETRY_EXC_ERR:
return IB_WC_RETRY_EXC_ERR;
case FLUSH_FATAL_ERR:
@@ -2464,6 +2699,64 @@
}
}
+static inline void
+set_ib_wc_op_sq(struct irdma_cq_poll_info *cq_poll_info,
+ struct ib_wc *entry)
+{
+ struct irdma_sc_qp *qp;
+
+ switch (cq_poll_info->op_type) {
+ case IRDMA_OP_TYPE_RDMA_WRITE:
+ case IRDMA_OP_TYPE_RDMA_WRITE_SOL:
+ entry->opcode = IB_WC_RDMA_WRITE;
+ break;
+ case IRDMA_OP_TYPE_RDMA_READ_INV_STAG:
+ case IRDMA_OP_TYPE_RDMA_READ:
+ entry->opcode = IB_WC_RDMA_READ;
+ break;
+ case IRDMA_OP_TYPE_SEND_SOL:
+ case IRDMA_OP_TYPE_SEND_SOL_INV:
+ case IRDMA_OP_TYPE_SEND_INV:
+ case IRDMA_OP_TYPE_SEND:
+ entry->opcode = IB_WC_SEND;
+ break;
+ case IRDMA_OP_TYPE_FAST_REG_NSMR:
+ entry->opcode = IB_WC_REG_MR;
+ break;
+ case IRDMA_OP_TYPE_INV_STAG:
+ entry->opcode = IB_WC_LOCAL_INV;
+ break;
+ default:
+ qp = cq_poll_info->qp_handle;
+ ibdev_err(irdma_get_ibdev(qp->dev), "Invalid opcode = %d in CQE\n",
+ cq_poll_info->op_type);
+ entry->status = IB_WC_GENERAL_ERR;
+ }
+}
+
+static inline void
+set_ib_wc_op_rq(struct irdma_cq_poll_info *cq_poll_info,
+ struct ib_wc *entry, bool send_imm_support)
+{
+ /**
+ * iWARP does not support sendImm, so the presence of Imm data
+ * must be WriteImm.
+ */
+ if (!send_imm_support) {
+ entry->opcode = cq_poll_info->imm_valid ?
+ IB_WC_RECV_RDMA_WITH_IMM : IB_WC_RECV;
+ return;
+ }
+ switch (cq_poll_info->op_type) {
+ case IB_OPCODE_RDMA_WRITE_ONLY_WITH_IMMEDIATE:
+ case IB_OPCODE_RDMA_WRITE_LAST_WITH_IMMEDIATE:
+ entry->opcode = IB_WC_RECV_RDMA_WITH_IMM;
+ break;
+ default:
+ entry->opcode = IB_WC_RECV;
+ }
+}
+
/**
* irdma_process_cqe - process cqe info
* @entry: processed cqe
@@ -2512,42 +2805,17 @@
}
}
- switch (cq_poll_info->op_type) {
- case IRDMA_OP_TYPE_RDMA_WRITE:
- case IRDMA_OP_TYPE_RDMA_WRITE_SOL:
- entry->opcode = IB_WC_RDMA_WRITE;
- break;
- case IRDMA_OP_TYPE_RDMA_READ_INV_STAG:
- case IRDMA_OP_TYPE_RDMA_READ:
- entry->opcode = IB_WC_RDMA_READ;
- break;
- case IRDMA_OP_TYPE_SEND_INV:
- case IRDMA_OP_TYPE_SEND_SOL:
- case IRDMA_OP_TYPE_SEND_SOL_INV:
- case IRDMA_OP_TYPE_SEND:
- entry->opcode = IB_WC_SEND;
- break;
- case IRDMA_OP_TYPE_FAST_REG_NSMR:
- entry->opcode = IB_WC_REG_MR;
- break;
- case IRDMA_OP_TYPE_INV_STAG:
- entry->opcode = IB_WC_LOCAL_INV;
- break;
- case IRDMA_OP_TYPE_REC_IMM:
- case IRDMA_OP_TYPE_REC:
- entry->opcode = cq_poll_info->op_type == IRDMA_OP_TYPE_REC_IMM ?
- IB_WC_RECV_RDMA_WITH_IMM : IB_WC_RECV;
+ if (cq_poll_info->q_type == IRDMA_CQE_QTYPE_SQ) {
+ set_ib_wc_op_sq(cq_poll_info, entry);
+ } else {
+ set_ib_wc_op_rq(cq_poll_info, entry,
+ qp->qp_uk.qp_caps & IRDMA_SEND_WITH_IMM ?
+ true : false);
if (qp->qp_uk.qp_type != IRDMA_QP_TYPE_ROCE_UD &&
cq_poll_info->stag_invalid_set) {
entry->ex.invalidate_rkey = cq_poll_info->inv_stag;
entry->wc_flags |= IB_WC_WITH_INVALIDATE;
}
- break;
- default:
- ibdev_err(&iwqp->iwdev->ibdev,
- "Invalid opcode = %d in CQE\n", cq_poll_info->op_type);
- entry->status = IB_WC_GENERAL_ERR;
- return;
}
if (qp->qp_uk.qp_type == IRDMA_QP_TYPE_ROCE_UD) {
@@ -2729,73 +2997,19 @@
promo_event = true;
}
- if (!iwcq->armed || promo_event) {
- iwcq->armed = true;
+ if (!atomic_cmpxchg(&iwcq->armed, 0, 1) || promo_event) {
iwcq->last_notify = cq_notify;
irdma_uk_cq_request_notification(ukcq, cq_notify);
}
- if ((notify_flags & IB_CQ_REPORT_MISSED_EVENTS) && !irdma_cq_empty(iwcq))
+ if ((notify_flags & IB_CQ_REPORT_MISSED_EVENTS) &&
+ (!irdma_cq_empty(iwcq) || !list_empty(&iwcq->cmpl_generated)))
ret = 1;
spin_unlock_irqrestore(&iwcq->lock, flags);
return ret;
}
-const char *const irdma_hw_stat_names[] = {
- /* gen1 - 32-bit */
- [IRDMA_HW_STAT_INDEX_IP4RXDISCARD] = "ip4InDiscards",
- [IRDMA_HW_STAT_INDEX_IP4RXTRUNC] = "ip4InTruncatedPkts",
- [IRDMA_HW_STAT_INDEX_IP4TXNOROUTE] = "ip4OutNoRoutes",
- [IRDMA_HW_STAT_INDEX_IP6RXDISCARD] = "ip6InDiscards",
- [IRDMA_HW_STAT_INDEX_IP6RXTRUNC] = "ip6InTruncatedPkts",
- [IRDMA_HW_STAT_INDEX_IP6TXNOROUTE] = "ip6OutNoRoutes",
- [IRDMA_HW_STAT_INDEX_TCPRTXSEG] = "tcpRetransSegs",
- [IRDMA_HW_STAT_INDEX_TCPRXOPTERR] = "tcpInOptErrors",
- [IRDMA_HW_STAT_INDEX_TCPRXPROTOERR] = "tcpInProtoErrors",
- [IRDMA_HW_STAT_INDEX_RXVLANERR] = "rxVlanErrors",
- /* gen1 - 64-bit */
- [IRDMA_HW_STAT_INDEX_IP4RXOCTS] = "ip4InOctets",
- [IRDMA_HW_STAT_INDEX_IP4RXPKTS] = "ip4InPkts",
- [IRDMA_HW_STAT_INDEX_IP4RXFRAGS] = "ip4InReasmRqd",
- [IRDMA_HW_STAT_INDEX_IP4RXMCPKTS] = "ip4InMcastPkts",
- [IRDMA_HW_STAT_INDEX_IP4TXOCTS] = "ip4OutOctets",
- [IRDMA_HW_STAT_INDEX_IP4TXPKTS] = "ip4OutPkts",
- [IRDMA_HW_STAT_INDEX_IP4TXFRAGS] = "ip4OutSegRqd",
- [IRDMA_HW_STAT_INDEX_IP4TXMCPKTS] = "ip4OutMcastPkts",
- [IRDMA_HW_STAT_INDEX_IP6RXOCTS] = "ip6InOctets",
- [IRDMA_HW_STAT_INDEX_IP6RXPKTS] = "ip6InPkts",
- [IRDMA_HW_STAT_INDEX_IP6RXFRAGS] = "ip6InReasmRqd",
- [IRDMA_HW_STAT_INDEX_IP6RXMCPKTS] = "ip6InMcastPkts",
- [IRDMA_HW_STAT_INDEX_IP6TXOCTS] = "ip6OutOctets",
- [IRDMA_HW_STAT_INDEX_IP6TXPKTS] = "ip6OutPkts",
- [IRDMA_HW_STAT_INDEX_IP6TXFRAGS] = "ip6OutSegRqd",
- [IRDMA_HW_STAT_INDEX_IP6TXMCPKTS] = "ip6OutMcastPkts",
- [IRDMA_HW_STAT_INDEX_TCPRXSEGS] = "tcpInSegs",
- [IRDMA_HW_STAT_INDEX_TCPTXSEG] = "tcpOutSegs",
- [IRDMA_HW_STAT_INDEX_RDMARXRDS] = "iwInRdmaReads",
- [IRDMA_HW_STAT_INDEX_RDMARXSNDS] = "iwInRdmaSends",
- [IRDMA_HW_STAT_INDEX_RDMARXWRS] = "iwInRdmaWrites",
- [IRDMA_HW_STAT_INDEX_RDMATXRDS] = "iwOutRdmaReads",
- [IRDMA_HW_STAT_INDEX_RDMATXSNDS] = "iwOutRdmaSends",
- [IRDMA_HW_STAT_INDEX_RDMATXWRS] = "iwOutRdmaWrites",
- [IRDMA_HW_STAT_INDEX_RDMAVBND] = "iwRdmaBnd",
- [IRDMA_HW_STAT_INDEX_RDMAVINV] = "iwRdmaInv",
-
- /* gen2 - 32-bit */
- [IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED] = "cnpHandled",
- [IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED] = "cnpIgnored",
- [IRDMA_HW_STAT_INDEX_TXNPCNPSENT] = "cnpSent",
- /* gen2 - 64-bit */
- [IRDMA_HW_STAT_INDEX_IP4RXMCOCTS] = "ip4InMcastOctets",
- [IRDMA_HW_STAT_INDEX_IP4TXMCOCTS] = "ip4OutMcastOctets",
- [IRDMA_HW_STAT_INDEX_IP6RXMCOCTS] = "ip6InMcastOctets",
- [IRDMA_HW_STAT_INDEX_IP6TXMCOCTS] = "ip6OutMcastOctets",
- [IRDMA_HW_STAT_INDEX_UDPRXPKTS] = "RxUDP",
- [IRDMA_HW_STAT_INDEX_UDPTXPKTS] = "TxUDP",
- [IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS] = "RxECNMrkd",
-};
-
/**
* mcast_list_add - Add a new mcast item to list
* @rf: RDMA PCI function
@@ -2963,7 +3177,7 @@
mc_qht_elem->mc_grp_ctx.vlan_id = vlan_id;
if (vlan_id < VLAN_N_VID)
mc_qht_elem->mc_grp_ctx.vlan_valid = true;
- mc_qht_elem->mc_grp_ctx.hmc_fcn_id = iwdev->vsi.fcn_id;
+ mc_qht_elem->mc_grp_ctx.hmc_fcn_id = iwdev->rf->sc_dev.hmc_fn_id;
mc_qht_elem->mc_grp_ctx.qs_handle =
iwqp->sc_qp.vsi->qos[iwqp->sc_qp.user_pri].qs_handle;
ether_addr_copy(mc_qht_elem->mc_grp_ctx.dest_mac_addr, dmac);
@@ -3115,7 +3329,7 @@
}
static __be64 irdma_mac_to_guid(struct ifnet *ndev){
- unsigned char *mac = IF_LLADDR(ndev);
+ const unsigned char *mac = IF_LLADDR(ndev);
__be64 guid;
unsigned char *dst = (unsigned char *)&guid;
@@ -3186,19 +3400,25 @@
dev_ops->modify_port = irdma_modify_port;
dev_ops->query_qp = irdma_query_qp;
dev_ops->reg_user_mr = irdma_reg_user_mr;
+ dev_ops->rereg_user_mr = irdma_rereg_user_mr;
dev_ops->req_notify_cq = irdma_req_notify_cq;
dev_ops->resize_cq = irdma_resize_cq;
}
static void
-irdma_set_device_roce_ops(struct ib_device *ibdev)
+irdma_set_device_mcast_ops(struct ib_device *ibdev)
{
struct ib_device *dev_ops = ibdev;
-
dev_ops->attach_mcast = irdma_attach_mcast;
+ dev_ops->detach_mcast = irdma_detach_mcast;
+}
+
+static void
+irdma_set_device_roce_ops(struct ib_device *ibdev)
+{
+ struct ib_device *dev_ops = ibdev;
dev_ops->create_ah = irdma_create_ah;
dev_ops->destroy_ah = irdma_destroy_ah;
- dev_ops->detach_mcast = irdma_detach_mcast;
dev_ops->get_link_layer = irdma_get_link_layer;
dev_ops->get_port_immutable = irdma_roce_port_immutable;
dev_ops->modify_qp = irdma_modify_qp_roce;
@@ -3226,6 +3446,11 @@
dev_ops->query_pkey = irdma_iw_query_pkey;
}
+static inline void
+irdma_set_device_gen1_ops(struct ib_device *ibdev)
+{
+}
+
/**
* irdma_init_roce_device - initialization of roce rdma device
* @iwdev: irdma device
@@ -3237,6 +3462,8 @@
iwdev->ibdev.node_type = RDMA_NODE_IB_CA;
iwdev->ibdev.node_guid = irdma_mac_to_guid(iwdev->netdev);
irdma_set_device_roce_ops(&iwdev->ibdev);
+ if (iwdev->rf->rdma_ver == IRDMA_GEN_2)
+ irdma_set_device_mcast_ops(&iwdev->ibdev);
}
/**
@@ -3290,11 +3517,14 @@
if (ret)
return ret;
}
+
iwdev->ibdev.phys_port_cnt = 1;
iwdev->ibdev.num_comp_vectors = iwdev->rf->ceqs_count;
iwdev->ibdev.dev.parent = iwdev->rf->dev_ctx.dev;
set_ibdev_dma_device(iwdev->ibdev, &pcidev->dev);
irdma_set_device_ops(&iwdev->ibdev);
+ if (iwdev->rf->rdma_ver == IRDMA_GEN_1)
+ irdma_set_device_gen1_ops(&iwdev->ibdev);
return 0;
}
@@ -3326,6 +3556,7 @@
iwdev->iw_status = 0;
irdma_port_ibevent(iwdev);
ib_unregister_device(&iwdev->ibdev);
+ dev_put(iwdev->netdev);
kfree(iwdev->ibdev.iwcm);
iwdev->ibdev.iwcm = NULL;
}
@@ -3343,6 +3574,7 @@
if (ret)
return ret;
+ dev_hold(iwdev->netdev);
sprintf(iwdev->ibdev.name, "irdma-%s", if_name(iwdev->netdev));
ret = ib_register_device(&iwdev->ibdev, NULL);
if (ret)
@@ -3356,9 +3588,7 @@
error:
kfree(iwdev->ibdev.iwcm);
iwdev->ibdev.iwcm = NULL;
- if (ret)
- irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,
- "Register RDMA device fail\n");
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "Register RDMA device fail\n");
return ret;
}
diff --git a/sys/dev/irdma/irdma_ws.c b/sys/dev/irdma/irdma_ws.c
--- a/sys/dev/irdma/irdma_ws.c
+++ b/sys/dev/irdma/irdma_ws.c
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2017 - 2021 Intel Corporation
+ * Copyright (c) 2017 - 2022 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -59,7 +59,7 @@
u16 node_index = 0;
ws_mem.size = sizeof(struct irdma_ws_node);
- ws_mem.va = kzalloc(ws_mem.size, GFP_ATOMIC);
+ ws_mem.va = kzalloc(ws_mem.size, GFP_KERNEL);
if (!ws_mem.va)
return NULL;
diff --git a/sys/dev/irdma/osdep.h b/sys/dev/irdma/osdep.h
--- a/sys/dev/irdma/osdep.h
+++ b/sys/dev/irdma/osdep.h
@@ -133,7 +133,7 @@
#define irdma_dev_warn(a, b, ...) printf(b, ##__VA_ARGS__) /*dev_warn(a, b)*/
#define irdma_dev_info(a, b, ...) printf(b, ##__VA_ARGS__)
#define irdma_pr_warn printf
-#define ibdev_err(ibdev, fmt, ...) irdma_dev_err(&((ibdev)->dev), fmt, ##__VA_ARGS__)
+#define ibdev_err(ibdev, fmt, ...) printf("%s:"fmt, (ibdev)->name, ##__VA_ARGS__)
#define dump_struct(s, sz, name) \
do { \
@@ -183,8 +183,10 @@
#define irdma_usec_delay(x) DELAY(x)
#define mdelay(x) DELAY((x) * 1000)
-#define rt_tos2priority(tos) (((tos >> 1) & 0x8 >> 1) | ((tos >> 2) ^ ((tos >> 3) << 1)))
+#define rt_tos2priority(tos) (tos >> 5)
#define ah_attr_to_dmac(attr) ((attr).dmac)
+#define kc_ib_modify_qp_is_ok(cur_state, next_state, type, mask, ll) \
+ ib_modify_qp_is_ok(cur_state, next_state, type, mask)
#define kc_rdma_gid_attr_network_type(sgid_attr, gid_type, gid) \
ib_gid_to_network_type(gid_type, gid)
#define irdma_del_timer_compat(tt) del_timer((tt))

File Metadata

Mime Type
text/plain
Expires
Sat, Feb 22, 1:47 PM (3 h, 29 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
16772583
Default Alt Text
D36944.diff (522 KB)

Event Timeline