Page MenuHomeFreeBSD
Files F110191686

D41296.diff
No OneTemporary
Actions

D41296.diff
View Options

diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64
--- a/sys/conf/files.arm64
+++ b/sys/conf/files.arm64
@@ -202,6 +202,8 @@
dev/cpufreq/cpufreq_dt.c optional cpufreq fdt
dev/dpaa2/dpaa2_bp.c optional soc_nxp_ls dpaa2
+dev/dpaa2/dpaa2_buf.c optional soc_nxp_ls dpaa2
+dev/dpaa2/dpaa2_channel.c optional soc_nxp_ls dpaa2
dev/dpaa2/dpaa2_cmd_if.m optional soc_nxp_ls dpaa2
dev/dpaa2/dpaa2_con.c optional soc_nxp_ls dpaa2
dev/dpaa2/dpaa2_console.c optional soc_nxp_ls dpaa2 fdt
@@ -216,6 +218,7 @@
dev/dpaa2/dpaa2_rc.c optional soc_nxp_ls dpaa2
dev/dpaa2/dpaa2_swp.c optional soc_nxp_ls dpaa2
dev/dpaa2/dpaa2_swp_if.m optional soc_nxp_ls dpaa2
+dev/dpaa2/dpaa2_types.c optional soc_nxp_ls dpaa2
dev/dpaa2/memac_mdio_acpi.c optional soc_nxp_ls dpaa2 acpi
dev/dpaa2/memac_mdio_common.c optional soc_nxp_ls dpaa2 acpi | soc_nxp_ls dpaa2 fdt
dev/dpaa2/memac_mdio_fdt.c optional soc_nxp_ls dpaa2 fdt
diff --git a/sys/dev/dpaa2/dpaa2_buf.h b/sys/dev/dpaa2/dpaa2_buf.h
new file mode 100644
--- /dev/null
+++ b/sys/dev/dpaa2/dpaa2_buf.h
@@ -0,0 +1,173 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright © 2023 Dmitry Salychev
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _DPAA2_BUF_H
+#define _DPAA2_BUF_H
+
+#include <sys/types.h>
+#include <sys/param.h>
+#include <sys/malloc.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+
+#include <machine/bus.h>
+
+#define DPAA2_RX_BUF_SIZE (MJUM9BYTES)
+
+struct dpaa2_buf {
+ bus_addr_t paddr;
+ caddr_t vaddr;
+ bus_dma_tag_t dmat;
+ bus_dmamap_t dmap;
+ bus_dma_segment_t seg;
+ int nseg;
+ struct mbuf *m;
+ struct dpaa2_buf *sgt;
+ void *opt;
+};
+
+#define DPAA2_BUF_INIT_TAGOPT(__buf, __tag, __opt) do { \
+ KASSERT((__buf) != NULL, ("%s: buf is NULL", __func__)); \
+ \
+ (__buf)->paddr = 0; \
+ (__buf)->vaddr = NULL; \
+ (__buf)->dmat = (__tag); \
+ (__buf)->dmap = NULL; \
+ (__buf)->seg.ds_addr = 0; \
+ (__buf)->seg.ds_len = 0; \
+ (__buf)->nseg = 0; \
+ (__buf)->m = NULL; \
+ (__buf)->sgt = NULL; \
+ (__buf)->opt = (__opt); \
+} while(0)
+#define DPAA2_BUF_INIT(__buf) DPAA2_BUF_INIT_TAGOPT((__buf), NULL, NULL)
+
+#if defined(INVARIANTS)
+/*
+ * TXPREP/TXREADY macros allow to verify whether Tx buffer is prepared to be
+ * seeded and/or ready to be used for transmission.
+ *
+ * NOTE: Any modification should be carefully analyzed and justified.
+ */
+#define DPAA2_BUF_ASSERT_TXPREP(__buf) do { \
+ struct dpaa2_buf *__sgt = (__buf)->sgt; \
+ KASSERT((__sgt) != NULL, ("%s: no S/G table?", __func__)); \
+ \
+ KASSERT((__buf)->paddr == 0, ("%s: paddr set?", __func__)); \
+ KASSERT((__buf)->vaddr == NULL, ("%s: vaddr set?", __func__)); \
+ KASSERT((__buf)->dmat != NULL, ("%s: no DMA tag?", __func__)); \
+ KASSERT((__buf)->dmap == NULL, ("%s: DMA map set?", __func__)); \
+ KASSERT((__buf)->seg.ds_addr == 0, ("%s: already mapped?", __func__)); \
+ KASSERT((__buf)->seg.ds_len == 0, ("%s: already mapped?", __func__)); \
+ KASSERT((__buf)->nseg == 0, ("%s: nseg > 0?", __func__)); \
+ KASSERT((__buf)->m == NULL, ("%s: mbuf set?", __func__)); \
+ KASSERT((__buf)->opt != NULL, ("%s: no Tx ring?", __func__)); \
+ \
+ KASSERT((__sgt)->paddr == 0, ("%s: S/G paddr set?", __func__)); \
+ KASSERT((__sgt)->vaddr == NULL, ("%s: S/G vaddr set?", __func__)); \
+ KASSERT((__sgt)->dmat != NULL, ("%s: no S/G DMA tag?", __func__)); \
+ KASSERT((__sgt)->dmap == NULL, ("%s: S/G DMA map set?", __func__)); \
+ KASSERT((__sgt)->seg.ds_addr == 0, ("%s: S/G mapped?", __func__)); \
+ KASSERT((__sgt)->seg.ds_len == 0, ("%s: S/G mapped?", __func__)); \
+ KASSERT((__sgt)->nseg == 0, ("%s: S/G nseg > 0?", __func__)); \
+ KASSERT((__sgt)->m == NULL, ("%s: S/G mbuf set?", __func__)); \
+ KASSERT((__sgt)->opt == (__buf),("%s: buf not linked?", __func__)); \
+} while(0)
+#define DPAA2_BUF_ASSERT_TXREADY(__buf) do { \
+ struct dpaa2_buf *__sgt = (__buf)->sgt; \
+ KASSERT((__sgt) != NULL, ("%s: no S/G table?", __func__)); \
+ \
+ KASSERT((__buf)->paddr == 0, ("%s: paddr set?", __func__)); \
+ KASSERT((__buf)->vaddr == NULL, ("%s: vaddr set?", __func__)); \
+ KASSERT((__buf)->dmat != NULL, ("%s: no DMA tag?", __func__)); \
+ KASSERT((__buf)->dmap != NULL, ("%s: no DMA map?", __func__)); \
+ KASSERT((__buf)->seg.ds_addr == 0, ("%s: already mapped?", __func__)); \
+ KASSERT((__buf)->seg.ds_len == 0, ("%s: already mapped?", __func__)); \
+ KASSERT((__buf)->nseg == 0, ("%s: nseg > 0?", __func__)); \
+ KASSERT((__buf)->m == NULL, ("%s: mbuf set?", __func__)); \
+ KASSERT((__buf)->opt != NULL, ("%s: no Tx ring?", __func__)); \
+ \
+ KASSERT((__sgt)->paddr == 0, ("%s: S/G paddr set?", __func__)); \
+ KASSERT((__sgt)->vaddr != NULL, ("%s: no S/G vaddr?", __func__)); \
+ KASSERT((__sgt)->dmat != NULL, ("%s: no S/G DMA tag?", __func__)); \
+ KASSERT((__sgt)->dmap != NULL, ("%s: no S/G DMA map?", __func__)); \
+ KASSERT((__sgt)->seg.ds_addr == 0, ("%s: S/G mapped?", __func__)); \
+ KASSERT((__sgt)->seg.ds_len == 0, ("%s: S/G mapped?", __func__)); \
+ KASSERT((__sgt)->nseg == 0, ("%s: S/G nseg > 0?", __func__)); \
+ KASSERT((__sgt)->m == NULL, ("%s: S/G mbuf set?", __func__)); \
+ KASSERT((__sgt)->opt == (__buf),("%s: buf not linked?", __func__)); \
+} while(0)
+#else /* !INVARIANTS */
+#define DPAA2_BUF_ASSERT_TXPREP(__buf) do { \
+} while(0)
+#define DPAA2_BUF_ASSERT_TXREADY(__buf) do { \
+} while(0)
+#endif /* INVARIANTS */
+
+#if defined(INVARIANTS)
+/*
+ * RXPREP/RXREADY macros allow to verify whether Rx buffer is prepared to be
+ * seeded and/or ready to be used for reception.
+ *
+ * NOTE: Any modification should be carefully analyzed and justified.
+ */
+#define DPAA2_BUF_ASSERT_RXPREP(__buf) do { \
+ KASSERT((__buf)->paddr == 0, ("%s: paddr set?", __func__)); \
+ KASSERT((__buf)->vaddr == NULL, ("%s: vaddr set?", __func__)); \
+ KASSERT((__buf)->dmat != NULL, ("%s: no DMA tag?", __func__)); \
+ /* KASSERT((__buf)->dmap == NULL, ("%s: DMA map set?", __func__)); */ \
+ KASSERT((__buf)->seg.ds_addr == 0, ("%s: already mapped?", __func__)); \
+ KASSERT((__buf)->seg.ds_len == 0, ("%s: already mapped?", __func__)); \
+ KASSERT((__buf)->nseg == 0, ("%s: nseg > 0?", __func__)); \
+ KASSERT((__buf)->m == NULL, ("%s: mbuf set?", __func__)); \
+ KASSERT((__buf)->sgt == NULL, ("%s: S/G table set?", __func__)); \
+ KASSERT((__buf)->opt != NULL, ("%s: no channel?", __func__)); \
+} while(0)
+#define DPAA2_BUF_ASSERT_RXREADY(__buf) do { \
+ KASSERT((__buf)->paddr != 0, ("%s: paddr not set?", __func__)); \
+ KASSERT((__buf)->vaddr != NULL, ("%s: vaddr not set?", __func__)); \
+ KASSERT((__buf)->dmat != NULL, ("%s: no DMA tag?", __func__)); \
+ KASSERT((__buf)->dmap != NULL, ("%s: no DMA map?", __func__)); \
+ KASSERT((__buf)->seg.ds_addr != 0, ("%s: not mapped?", __func__)); \
+ KASSERT((__buf)->seg.ds_len != 0, ("%s: not mapped?", __func__)); \
+ KASSERT((__buf)->nseg == 1, ("%s: nseg != 1?", __func__)); \
+ KASSERT((__buf)->m != NULL, ("%s: no mbuf?", __func__)); \
+ KASSERT((__buf)->sgt == NULL, ("%s: S/G table set?", __func__)); \
+ KASSERT((__buf)->opt != NULL, ("%s: no channel?", __func__)); \
+} while(0)
+#else /* !INVARIANTS */
+#define DPAA2_BUF_ASSERT_RXPREP(__buf) do { \
+} while(0)
+#define DPAA2_BUF_ASSERT_RXREADY(__buf) do { \
+} while(0)
+#endif /* INVARIANTS */
+
+int dpaa2_buf_seed_pool(device_t, device_t, void *, uint32_t, int, struct mtx *);
+int dpaa2_buf_seed_rxb(device_t, struct dpaa2_buf *, int, struct mtx *);
+int dpaa2_buf_seed_txb(device_t, struct dpaa2_buf *);
+
+#endif /* _DPAA2_BUF_H */
diff --git a/sys/dev/dpaa2/dpaa2_buf.c b/sys/dev/dpaa2/dpaa2_buf.c
new file mode 100644
--- /dev/null
+++ b/sys/dev/dpaa2/dpaa2_buf.c
@@ -0,0 +1,246 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright © 2023 Dmitry Salychev
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/types.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/mbuf.h>
+#include <sys/malloc.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+
+#include <machine/bus.h>
+
+#include "dpaa2_types.h"
+#include "dpaa2_buf.h"
+#include "dpaa2_bp.h"
+#include "dpaa2_channel.h"
+#include "dpaa2_swp.h"
+#include "dpaa2_swp_if.h"
+#include "dpaa2_ni.h"
+
+MALLOC_DEFINE(M_DPAA2_RXB, "dpaa2_rxb", "DPAA2 DMA-mapped buffer (Rx)");
+
+/**
+ * @brief Allocate Rx buffers visible to QBMan and release them to the
+ * buffer pool.
+ */
+int
+dpaa2_buf_seed_pool(device_t dev, device_t bpdev, void *arg, uint32_t count,
+ int size, struct mtx *dma_mtx)
+{
+ struct dpaa2_ni_softc *sc = device_get_softc(dev);
+ struct dpaa2_bp_softc *bpsc = device_get_softc(bpdev);
+ struct dpaa2_channel *ch = (struct dpaa2_channel *)arg;
+ struct dpaa2_buf *buf;
+ const int alloc = DPAA2_ATOMIC_READ(&sc->buf_num);
+ const uint16_t bpid = bpsc->attr.bpid;
+ bus_addr_t paddr[DPAA2_SWP_BUFS_PER_CMD];
+ int error, bufn = 0;
+
+#if defined(INVARIANTS)
+ KASSERT(ch->rx_dmat != NULL, ("%s: no DMA tag?", __func__));
+ if (dma_mtx != NULL) {
+ mtx_assert(dma_mtx, MA_OWNED);
+ }
+#endif /* INVARIANTS */
+
+#ifdef _notyet_
+ /* Limit amount of buffers released to the pool */
+ count = (alloc + count > DPAA2_NI_BUFS_MAX)
+ ? DPAA2_NI_BUFS_MAX - alloc : count;
+#endif
+
+ /* Release "count" buffers to the pool */
+ for (int i = alloc; i < alloc + count; i++) {
+ /* Enough buffers were allocated for a single command */
+ if (bufn == DPAA2_SWP_BUFS_PER_CMD) {
+ error = DPAA2_SWP_RELEASE_BUFS(ch->io_dev, bpid, paddr,
+ bufn);
+ if (error) {
+ device_printf(sc->dev, "%s: failed to release "
+ "buffers to the pool (1)\n", __func__);
+ return (error);
+ }
+ DPAA2_ATOMIC_ADD(&sc->buf_num, bufn);
+ bufn = 0;
+ }
+
+ buf = malloc(sizeof(struct dpaa2_buf), M_DPAA2_RXB, M_NOWAIT);
+ if (buf == NULL) {
+ device_printf(dev, "%s: malloc() failed\n", __func__);
+ return (ENOMEM);
+ }
+ DPAA2_BUF_INIT_TAGOPT(buf, ch->rx_dmat, ch);
+
+ error = dpaa2_buf_seed_rxb(dev, buf, size, dma_mtx);
+ if (error != 0) {
+ device_printf(dev, "%s: dpaa2_buf_seed_rxb() failed: "
+ "error=%d/n", __func__, error);
+ break;
+ }
+ paddr[bufn] = buf->paddr;
+ bufn++;
+ }
+
+ /* Release reminder of the buffers to the pool */
+ if (bufn > 0) {
+ error = DPAA2_SWP_RELEASE_BUFS(ch->io_dev, bpid, paddr, bufn);
+ if (error) {
+ device_printf(sc->dev, "%s: failed to release "
+ "buffers to the pool (2)\n", __func__);
+ return (error);
+ }
+ DPAA2_ATOMIC_ADD(&sc->buf_num, bufn);
+ }
+
+ return (0);
+}
+
+/**
+ * @brief Prepare Rx buffer to be released to the buffer pool.
+ */
+int
+dpaa2_buf_seed_rxb(device_t dev, struct dpaa2_buf *buf, int size,
+ struct mtx *dma_mtx)
+{
+ struct dpaa2_ni_softc *sc = device_get_softc(dev);
+ struct dpaa2_fa *fa;
+ bool map_created = false;
+ bool mbuf_alloc = false;
+ int error;
+
+#if defined(INVARIANTS)
+ DPAA2_BUF_ASSERT_RXPREP(buf);
+ if (dma_mtx != NULL) {
+ mtx_assert(dma_mtx, MA_OWNED);
+ }
+#endif /* INVARIANTS */
+
+ if (__predict_false(buf->dmap == NULL)) {
+ error = bus_dmamap_create(buf->dmat, 0, &buf->dmap);
+ if (error != 0) {
+ device_printf(dev, "%s: failed to create DMA map: "
+ "error=%d\n", __func__, error);
+ goto fail_map_create;
+ }
+ map_created = true;
+ }
+
+ if (__predict_true(buf->m == NULL)) {
+ buf->m = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, size);
+ if (__predict_false(buf->m == NULL)) {
+ device_printf(dev, "%s: m_getjcl() failed\n", __func__);
+ error = ENOMEM;
+ goto fail_mbuf_alloc;
+ }
+ buf->m->m_len = buf->m->m_ext.ext_size;
+ buf->m->m_pkthdr.len = buf->m->m_ext.ext_size;
+ mbuf_alloc = true;
+ }
+
+ error = bus_dmamap_load_mbuf_sg(buf->dmat, buf->dmap, buf->m, &buf->seg,
+ &buf->nseg, BUS_DMA_NOWAIT);
+ KASSERT(buf->nseg == 1, ("%s: one segment expected: nseg=%d", __func__,
+ buf->nseg));
+ KASSERT(error == 0, ("%s: bus_dmamap_load_mbuf_sg() failed: error=%d",
+ __func__, error));
+ if (__predict_false(error != 0 || buf->nseg != 1)) {
+ device_printf(sc->dev, "%s: bus_dmamap_load_mbuf_sg() failed: "
+ "error=%d, nsegs=%d\n", __func__, error, buf->nseg);
+ goto fail_mbuf_map;
+ }
+ buf->paddr = buf->seg.ds_addr;
+ buf->vaddr = buf->m->m_data;
+
+ /* Populate frame annotation for future use */
+ fa = (struct dpaa2_fa *)buf->vaddr;
+ fa->magic = DPAA2_MAGIC;
+ fa->buf = buf;
+
+ bus_dmamap_sync(buf->dmat, buf->dmap, BUS_DMASYNC_PREREAD);
+
+ DPAA2_BUF_ASSERT_RXREADY(buf);
+
+ return (0);
+
+fail_mbuf_map:
+ if (mbuf_alloc) {
+ m_freem(buf->m);
+ buf->m = NULL;
+ }
+fail_mbuf_alloc:
+ if (map_created) {
+ (void)bus_dmamap_destroy(buf->dmat, buf->dmap);
+ }
+fail_map_create:
+ return (error);
+}
+
+/**
+ * @brief Prepare Tx buffer to be added to the Tx ring.
+ */
+int
+dpaa2_buf_seed_txb(device_t dev, struct dpaa2_buf *buf)
+{
+ struct dpaa2_buf *sgt = buf->sgt;
+ bool map_created = false;
+ int error;
+
+ DPAA2_BUF_ASSERT_TXPREP(buf);
+
+ if (buf->dmap == NULL) {
+ error = bus_dmamap_create(buf->dmat, 0, &buf->dmap);
+ if (error != 0) {
+ device_printf(dev, "%s: bus_dmamap_create() failed: "
+ "error=%d\n", __func__, error);
+ goto fail_map_create;
+ }
+ map_created = true;
+ }
+
+ if (sgt->vaddr == NULL) {
+ error = bus_dmamem_alloc(sgt->dmat, (void **)&sgt->vaddr,
+ BUS_DMA_ZERO | BUS_DMA_COHERENT, &sgt->dmap);
+ if (error != 0) {
+ device_printf(dev, "%s: bus_dmamem_alloc() failed: "
+ "error=%d\n", __func__, error);
+ goto fail_mem_alloc;
+ }
+ }
+
+ DPAA2_BUF_ASSERT_TXREADY(buf);
+
+ return (0);
+
+fail_mem_alloc:
+ if (map_created) {
+ (void)bus_dmamap_destroy(buf->dmat, buf->dmap);
+ }
+fail_map_create:
+ return (error);
+}
diff --git a/sys/dev/dpaa2/dpaa2_channel.h b/sys/dev/dpaa2/dpaa2_channel.h
new file mode 100644
--- /dev/null
+++ b/sys/dev/dpaa2/dpaa2_channel.h
@@ -0,0 +1,95 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright © 2023 Dmitry Salychev
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _DPAA2_CHANNEL_H
+#define _DPAA2_CHANNEL_H
+
+#include <sys/types.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/queue.h>
+#include <sys/taskqueue.h>
+#include <sys/buf_ring.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+
+#include "dpaa2_types.h"
+#include "dpaa2_io.h"
+#include "dpaa2_ni.h"
+
+#define DPAA2_TX_BUFRING_SZ (4096u)
+
+/**
+ * @brief QBMan channel to process ingress traffic.
+ *
+ * NOTE: Several WQs are organized into a single channel.
+ */
+struct dpaa2_channel {
+ device_t ni_dev;
+ device_t io_dev;
+ device_t con_dev;
+ uint16_t id;
+ uint16_t flowid;
+
+ uint64_t tx_frames;
+ uint64_t tx_dropped;
+
+ struct mtx dma_mtx;
+ bus_dma_tag_t rx_dmat;
+ bus_dma_tag_t tx_dmat;
+ bus_dma_tag_t sgt_dmat;
+
+ struct dpaa2_io_notif_ctx ctx; /* to configure CDANs */
+
+ struct dpaa2_buf store; /* to keep VDQ responses */
+ uint32_t store_sz; /* in frames */
+ uint32_t store_idx; /* frame index */
+
+ uint32_t recycled_n;
+ struct dpaa2_buf *recycled[DPAA2_SWP_BUFS_PER_CMD];
+
+ uint32_t rxq_n;
+ struct dpaa2_ni_fq rx_queues[DPAA2_MAX_TCS];
+ struct dpaa2_ni_fq txc_queue;
+
+ struct taskqueue *cleanup_tq;
+ struct task cleanup_task;
+ struct task bp_task;
+
+ struct mtx xmit_mtx;
+ struct buf_ring *xmit_br;
+} __aligned(CACHE_LINE_SIZE);
+
+int dpaa2_chan_setup(device_t, device_t, device_t, device_t,
+ struct dpaa2_channel **, uint32_t, task_fn_t);
+int dpaa2_chan_setup_fq(device_t, struct dpaa2_channel *,
+ enum dpaa2_ni_queue_type);
+int dpaa2_chan_next_frame(struct dpaa2_channel *, struct dpaa2_dq **);
+
+#endif /* _DPAA2_CHANNEL_H */
diff --git a/sys/dev/dpaa2/dpaa2_channel.c b/sys/dev/dpaa2/dpaa2_channel.c
new file mode 100644
--- /dev/null
+++ b/sys/dev/dpaa2/dpaa2_channel.c
@@ -0,0 +1,557 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright © 2023 Dmitry Salychev
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * QBMan channel to process ingress traffic (Rx, Tx confirmation, Rx error).
+ *
+ * NOTE: Several WQs are organized into a single channel.
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/bus.h>
+#include <sys/rman.h>
+#include <sys/module.h>
+#include <sys/malloc.h>
+#include <sys/mutex.h>
+#include <sys/socket.h>
+#include <sys/sockio.h>
+#include <sys/sysctl.h>
+#include <sys/mbuf.h>
+#include <sys/taskqueue.h>
+#include <sys/sysctl.h>
+#include <sys/buf_ring.h>
+#include <sys/smp.h>
+#include <sys/proc.h>
+
+#include <machine/bus.h>
+#include <machine/resource.h>
+#include <machine/atomic.h>
+#include <machine/vmparam.h>
+
+#include <net/ethernet.h>
+#include <net/bpf.h>
+#include <net/if.h>
+#include <net/if_dl.h>
+#include <net/if_media.h>
+#include <net/if_types.h>
+#include <net/if_var.h>
+
+#include "dpaa2_types.h"
+#include "dpaa2_channel.h"
+#include "dpaa2_ni.h"
+#include "dpaa2_mc.h"
+#include "dpaa2_mc_if.h"
+#include "dpaa2_mcp.h"
+#include "dpaa2_io.h"
+#include "dpaa2_con.h"
+#include "dpaa2_buf.h"
+#include "dpaa2_swp.h"
+#include "dpaa2_swp_if.h"
+#include "dpaa2_bp.h"
+#include "dpaa2_cmd_if.h"
+
+MALLOC_DEFINE(M_DPAA2_CH, "dpaa2_ch", "DPAA2 QBMan Channel");
+
+#define RX_SEG_N (1u)
+#define RX_SEG_SZ (((MJUM9BYTES - 1) / PAGE_SIZE + 1) * PAGE_SIZE)
+#define RX_SEG_MAXSZ (((MJUM9BYTES - 1) / PAGE_SIZE + 1) * PAGE_SIZE)
+CTASSERT(RX_SEG_SZ % PAGE_SIZE == 0);
+CTASSERT(RX_SEG_MAXSZ % PAGE_SIZE == 0);
+
+#define TX_SEG_N (16u) /* XXX-DSL: does DPAA2 limit exist? */
+#define TX_SEG_SZ (PAGE_SIZE)
+#define TX_SEG_MAXSZ (TX_SEG_N * TX_SEG_SZ)
+CTASSERT(TX_SEG_SZ % PAGE_SIZE == 0);
+CTASSERT(TX_SEG_MAXSZ % PAGE_SIZE == 0);
+
+#define SGT_SEG_N (1u)
+#define SGT_SEG_SZ (PAGE_SIZE)
+#define SGT_SEG_MAXSZ (PAGE_SIZE)
+CTASSERT(SGT_SEG_SZ % PAGE_SIZE == 0);
+CTASSERT(SGT_SEG_MAXSZ % PAGE_SIZE == 0);
+
+static int dpaa2_chan_setup_dma(device_t, struct dpaa2_channel *, bus_size_t);
+static int dpaa2_chan_alloc_storage(device_t, struct dpaa2_channel *, bus_size_t,
+ int, bus_size_t);
+static void dpaa2_chan_bp_task(void *, int);
+
+/**
+ * @brief Сonfigures QBMan channel and registers data availability notifications.
+ */
+int
+dpaa2_chan_setup(device_t dev, device_t iodev, device_t condev, device_t bpdev,
+ struct dpaa2_channel **channel, uint32_t flowid, task_fn_t cleanup_task_fn)
+{
+ device_t pdev = device_get_parent(dev);
+ device_t child = dev;
+ struct dpaa2_ni_softc *sc = device_get_softc(dev);
+ struct dpaa2_io_softc *iosc = device_get_softc(iodev);
+ struct dpaa2_con_softc *consc = device_get_softc(condev);
+ struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev);
+ struct dpaa2_devinfo *ioinfo = device_get_ivars(iodev);
+ struct dpaa2_devinfo *coninfo = device_get_ivars(condev);
+ struct dpaa2_con_notif_cfg notif_cfg;
+ struct dpaa2_io_notif_ctx *ctx;
+ struct dpaa2_channel *ch = NULL;
+ struct dpaa2_cmd cmd;
+ uint16_t rctk, contk;
+ int error;
+
+ DPAA2_CMD_INIT(&cmd);
+
+ error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rctk);
+ if (error) {
+ device_printf(dev, "%s: failed to open DPRC: id=%d, error=%d\n",
+ __func__, rcinfo->id, error);
+ goto fail_rc_open;
+ }
+ error = DPAA2_CMD_CON_OPEN(dev, child, &cmd, coninfo->id, &contk);
+ if (error) {
+ device_printf(dev, "%s: failed to open DPCON: id=%d, error=%d\n",
+ __func__, coninfo->id, error);
+ goto fail_con_open;
+ }
+
+ error = DPAA2_CMD_CON_ENABLE(dev, child, &cmd);
+ if (error) {
+ device_printf(dev, "%s: failed to enable channel: dpcon_id=%d, "
+ "chan_id=%d\n", __func__, coninfo->id, consc->attr.chan_id);
+ goto fail_con_enable;
+ }
+
+ ch = malloc(sizeof(struct dpaa2_channel), M_DPAA2_CH, M_WAITOK | M_ZERO);
+ if (ch == NULL) {
+ device_printf(dev, "%s: malloc() failed\n", __func__);
+ error = ENOMEM;
+ goto fail_malloc;
+ }
+
+ ch->ni_dev = dev;
+ ch->io_dev = iodev;
+ ch->con_dev = condev;
+ ch->id = consc->attr.chan_id;
+ ch->flowid = flowid;
+ ch->tx_frames = 0; /* for debug purposes */
+ ch->tx_dropped = 0; /* for debug purposes */
+ ch->store_sz = 0;
+ ch->store_idx = 0;
+ ch->recycled_n = 0;
+ ch->rxq_n = 0;
+
+ NET_TASK_INIT(&ch->cleanup_task, 0, cleanup_task_fn, ch);
+ NET_TASK_INIT(&ch->bp_task, 0, dpaa2_chan_bp_task, ch);
+
+ ch->cleanup_tq = taskqueue_create("dpaa2_ch cleanup", M_WAITOK,
+ taskqueue_thread_enqueue, &ch->cleanup_tq);
+ taskqueue_start_threads_cpuset(&ch->cleanup_tq, 1, PI_NET,
+ &iosc->cpu_mask, "dpaa2_ch%d cleanup", ch->id);
+
+ error = dpaa2_chan_setup_dma(dev, ch, sc->buf_align);
+ if (error != 0) {
+ device_printf(dev, "%s: failed to setup DMA\n", __func__);
+ goto fail_dma_setup;
+ }
+
+ mtx_init(&ch->xmit_mtx, "dpaa2_ch_xmit", NULL, MTX_DEF);
+
+ ch->xmit_br = buf_ring_alloc(DPAA2_TX_BUFRING_SZ, M_DEVBUF, M_NOWAIT,
+ &ch->xmit_mtx);
+ if (ch->xmit_br == NULL) {
+ device_printf(dev, "%s: buf_ring_alloc() failed\n", __func__);
+ error = ENOMEM;
+ goto fail_buf_ring;
+ }
+
+ DPAA2_BUF_INIT(&ch->store);
+
+ /* Register the new notification context */
+ ctx = &ch->ctx;
+ ctx->qman_ctx = (uint64_t)ctx;
+ ctx->cdan_en = true;
+ ctx->fq_chan_id = ch->id;
+ ctx->io_dev = ch->io_dev;
+ ctx->channel = ch;
+ error = DPAA2_SWP_CONF_WQ_CHANNEL(ch->io_dev, ctx);
+ if (error) {
+ device_printf(dev, "%s: failed to register CDAN context\n",
+ __func__);
+ goto fail_dpcon_notif;
+ }
+
+ /* Register DPCON notification within Management Complex */
+ notif_cfg.dpio_id = ioinfo->id;
+ notif_cfg.prior = 0;
+ notif_cfg.qman_ctx = ctx->qman_ctx;
+ error = DPAA2_CMD_CON_SET_NOTIF(dev, child, &cmd, &notif_cfg);
+ if (error) {
+ device_printf(dev, "%s: failed to register DPCON "
+ "notifications: dpcon_id=%d, chan_id=%d\n", __func__,
+ coninfo->id, consc->attr.chan_id);
+ goto fail_dpcon_notif;
+ }
+
+ /* Allocate initial # of Rx buffers and a channel storage */
+ error = dpaa2_buf_seed_pool(dev, bpdev, ch, DPAA2_NI_BUFS_INIT,
+ DPAA2_RX_BUF_SIZE, NULL);
+ if (error) {
+ device_printf(dev, "%s: failed to seed buffer pool\n",
+ __func__);
+ goto fail_dpcon_notif;
+ }
+ error = dpaa2_chan_alloc_storage(dev, ch, DPAA2_ETH_STORE_SIZE,
+ BUS_DMA_NOWAIT, sc->buf_align);
+ if (error != 0) {
+ device_printf(dev, "%s: failed to allocate channel storage\n",
+ __func__);
+ goto fail_dpcon_notif;
+ } else {
+ ch->store_sz = DPAA2_ETH_STORE_FRAMES;
+ }
+
+ /* Prepare queues for the channel */
+ error = dpaa2_chan_setup_fq(dev, ch, DPAA2_NI_QUEUE_TX_CONF);
+ if (error) {
+ device_printf(dev, "%s: failed to prepare TxConf queue: "
+ "error=%d\n", __func__, error);
+ goto fail_fq_setup;
+ }
+ error = dpaa2_chan_setup_fq(dev, ch, DPAA2_NI_QUEUE_RX);
+ if (error) {
+ device_printf(dev, "%s: failed to prepare Rx queue: error=%d\n",
+ __func__, error);
+ goto fail_fq_setup;
+ }
+
+ if (bootverbose) {
+ device_printf(dev, "channel: dpio_id=%d dpcon_id=%d chan_id=%d, "
+ "priorities=%d\n", ioinfo->id, coninfo->id, ch->id,
+ consc->attr.prior_num);
+ }
+
+ *channel = ch;
+
+ (void)DPAA2_CMD_CON_CLOSE(dev, child, &cmd);
+ (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rctk));
+
+ return (0);
+
+fail_fq_setup:
+ if (ch->store.vaddr != NULL) {
+ bus_dmamem_free(ch->store.dmat, ch->store.vaddr, ch->store.dmap);
+ }
+ if (ch->store.dmat != NULL) {
+ bus_dma_tag_destroy(ch->store.dmat);
+ }
+ ch->store.dmat = NULL;
+ ch->store.vaddr = NULL;
+ ch->store.paddr = 0;
+ ch->store.nseg = 0;
+fail_dpcon_notif:
+ buf_ring_free(ch->xmit_br, M_DEVBUF);
+fail_buf_ring:
+ mtx_destroy(&ch->xmit_mtx);
+fail_dma_setup:
+ /* while (taskqueue_cancel(ch->cleanup_tq, &ch->cleanup_task, NULL)) { */
+ /* taskqueue_drain(ch->cleanup_tq, &ch->cleanup_task); */
+ /* } */
+ /* taskqueue_free(ch->cleanup_tq); */
+fail_malloc:
+ (void)DPAA2_CMD_CON_DISABLE(dev, child, DPAA2_CMD_TK(&cmd, contk));
+fail_con_enable:
+ (void)DPAA2_CMD_CON_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, contk));
+fail_con_open:
+ (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rctk));
+fail_rc_open:
+ return (error);
+}
+
+/**
+ * @brief Performs an initial configuration of the frame queue.
+ */
+int
+dpaa2_chan_setup_fq(device_t dev, struct dpaa2_channel *ch,
+ enum dpaa2_ni_queue_type queue_type)
+{
+ struct dpaa2_ni_softc *sc = device_get_softc(dev);
+ struct dpaa2_ni_fq *fq;
+
+ switch (queue_type) {
+ case DPAA2_NI_QUEUE_TX_CONF:
+ /* One queue per channel */
+ fq = &ch->txc_queue;
+ fq->chan = ch;
+ fq->flowid = ch->flowid;
+ fq->tc = 0; /* ignored */
+ fq->type = queue_type;
+ break;
+ case DPAA2_NI_QUEUE_RX:
+ KASSERT(sc->attr.num.rx_tcs <= DPAA2_MAX_TCS,
+ ("too many Rx traffic classes: rx_tcs=%d\n",
+ sc->attr.num.rx_tcs));
+
+ /* One queue per Rx traffic class within a channel */
+ for (int i = 0; i < sc->attr.num.rx_tcs; i++) {
+ fq = &ch->rx_queues[i];
+ fq->chan = ch;
+ fq->flowid = ch->flowid;
+ fq->tc = (uint8_t) i;
+ fq->type = queue_type;
+
+ ch->rxq_n++;
+ }
+ break;
+ case DPAA2_NI_QUEUE_RX_ERR:
+ /* One queue per network interface */
+ fq = &sc->rxe_queue;
+ fq->chan = ch;
+ fq->flowid = 0; /* ignored */
+ fq->tc = 0; /* ignored */
+ fq->type = queue_type;
+ break;
+ default:
+ device_printf(dev, "%s: unexpected frame queue type: %d\n",
+ __func__, queue_type);
+ return (EINVAL);
+ }
+
+ return (0);
+}
+
+/**
+ * @brief Obtain the next dequeue response from the channel storage.
+ */
+int
+dpaa2_chan_next_frame(struct dpaa2_channel *ch, struct dpaa2_dq **dq)
+{
+ struct dpaa2_buf *buf = &ch->store;
+ struct dpaa2_dq *msgs = (struct dpaa2_dq *)buf->vaddr;
+ struct dpaa2_dq *msg = &msgs[ch->store_idx];
+ int rc = EINPROGRESS;
+
+ ch->store_idx++;
+
+ if (msg->fdr.desc.stat & DPAA2_DQ_STAT_EXPIRED) {
+ rc = EALREADY; /* VDQ command is expired */
+ ch->store_idx = 0;
+ if (!(msg->fdr.desc.stat & DPAA2_DQ_STAT_VALIDFRAME)) {
+ msg = NULL; /* Null response, FD is invalid */
+ }
+ }
+ if (msg != NULL && (msg->fdr.desc.stat & DPAA2_DQ_STAT_FQEMPTY)) {
+ rc = ENOENT; /* FQ is empty */
+ ch->store_idx = 0;
+ }
+
+ if (dq != NULL) {
+ *dq = msg;
+ }
+
+ return (rc);
+}
+
+static int
+dpaa2_chan_setup_dma(device_t dev, struct dpaa2_channel *ch,
+ bus_size_t alignment)
+{
+ int error;
+
+ mtx_init(&ch->dma_mtx, "dpaa2_ch_dma_mtx", NULL, MTX_DEF);
+
+ error = bus_dma_tag_create(
+ bus_get_dma_tag(dev), /* parent */
+ alignment, 0, /* alignment, boundary */
+ BUS_SPACE_MAXADDR, /* low restricted addr */
+ BUS_SPACE_MAXADDR, /* high restricted addr */
+ NULL, NULL, /* filter, filterarg */
+ RX_SEG_MAXSZ, /* maxsize */
+ RX_SEG_N, /* nsegments */
+ RX_SEG_SZ, /* maxsegsize */
+ 0, /* flags */
+ NULL, /* lockfunc */
+ NULL, /* lockarg */
+ &ch->rx_dmat);
+ if (error) {
+ device_printf(dev, "%s: failed to create rx_dmat\n", __func__);
+ goto fail_rx_tag;
+ }
+
+ error = bus_dma_tag_create(
+ bus_get_dma_tag(dev), /* parent */
+ alignment, 0, /* alignment, boundary */
+ BUS_SPACE_MAXADDR, /* low restricted addr */
+ BUS_SPACE_MAXADDR, /* high restricted addr */
+ NULL, NULL, /* filter, filterarg */
+ TX_SEG_MAXSZ, /* maxsize */
+ TX_SEG_N, /* nsegments */
+ TX_SEG_SZ, /* maxsegsize */
+ 0, /* flags */
+ NULL, /* lockfunc */
+ NULL, /* lockarg */
+ &ch->tx_dmat);
+ if (error) {
+ device_printf(dev, "%s: failed to create tx_dmat\n", __func__);
+ goto fail_tx_tag;
+ }
+
+ error = bus_dma_tag_create(
+ bus_get_dma_tag(dev), /* parent */
+ alignment, 0, /* alignment, boundary */
+ BUS_SPACE_MAXADDR, /* low restricted addr */
+ BUS_SPACE_MAXADDR, /* high restricted addr */
+ NULL, NULL, /* filter, filterarg */
+ SGT_SEG_MAXSZ, /* maxsize */
+ SGT_SEG_N, /* nsegments */
+ SGT_SEG_SZ, /* maxsegsize */
+ 0, /* flags */
+ NULL, /* lockfunc */
+ NULL, /* lockarg */
+ &ch->sgt_dmat);
+ if (error) {
+ device_printf(dev, "%s: failed to create sgt_dmat\n", __func__);
+ goto fail_sgt_tag;
+ }
+
+ return (0);
+
+fail_sgt_tag:
+ bus_dma_tag_destroy(ch->tx_dmat);
+fail_tx_tag:
+ bus_dma_tag_destroy(ch->rx_dmat);
+fail_rx_tag:
+ mtx_destroy(&ch->dma_mtx);
+ ch->rx_dmat = NULL;
+ ch->tx_dmat = NULL;
+ ch->sgt_dmat = NULL;
+
+ return (error);
+}
+
+/**
+ * @brief Allocate a DMA-mapped storage to keep responses from VDQ command.
+ */
+static int
+dpaa2_chan_alloc_storage(device_t dev, struct dpaa2_channel *ch, bus_size_t size,
+ int mapflags, bus_size_t alignment)
+{
+ struct dpaa2_buf *buf = &ch->store;
+ uint32_t maxsize = ((size - 1) / PAGE_SIZE + 1) * PAGE_SIZE;
+ int error;
+
+ error = bus_dma_tag_create(
+ bus_get_dma_tag(dev), /* parent */
+ alignment, 0, /* alignment, boundary */
+ BUS_SPACE_MAXADDR, /* low restricted addr */
+ BUS_SPACE_MAXADDR, /* high restricted addr */
+ NULL, NULL, /* filter, filterarg */
+ maxsize, /* maxsize */
+ 1, /* nsegments */
+ maxsize, /* maxsegsize */
+ BUS_DMA_ALLOCNOW, /* flags */
+ NULL, /* lockfunc */
+ NULL, /* lockarg */
+ &buf->dmat);
+ if (error != 0) {
+ device_printf(dev, "%s: failed to create DMA tag\n", __func__);
+ goto fail_tag;
+ }
+
+ error = bus_dmamem_alloc(buf->dmat, (void **)&buf->vaddr,
+ BUS_DMA_ZERO | BUS_DMA_COHERENT, &buf->dmap);
+ if (error != 0) {
+ device_printf(dev, "%s: failed to allocate storage memory\n",
+ __func__);
+ goto fail_map_create;
+ }
+
+ buf->paddr = 0;
+ error = bus_dmamap_load(buf->dmat, buf->dmap, buf->vaddr, size,
+ dpaa2_dmamap_oneseg_cb, &buf->paddr, mapflags);
+ if (error != 0) {
+ device_printf(dev, "%s: failed to map storage memory\n",
+ __func__);
+ goto fail_map_load;
+ }
+
+ bus_dmamap_sync(buf->dmat, buf->dmap,
+ BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
+
+ buf->nseg = 1;
+
+ return (0);
+
+fail_map_load:
+ bus_dmamem_free(buf->dmat, buf->vaddr, buf->dmap);
+fail_map_create:
+ bus_dma_tag_destroy(buf->dmat);
+fail_tag:
+ buf->dmat = NULL;
+ buf->vaddr = NULL;
+ buf->paddr = 0;
+ buf->nseg = 0;
+
+ return (error);
+}
+
+/**
+ * @brief Release new buffers to the buffer pool if necessary.
+ */
+static void
+dpaa2_chan_bp_task(void *arg, int count)
+{
+ struct dpaa2_channel *ch = (struct dpaa2_channel *)arg;
+ struct dpaa2_ni_softc *sc = device_get_softc(ch->ni_dev);
+ struct dpaa2_bp_softc *bpsc;
+ struct dpaa2_bp_conf bpconf;
+ const int buf_num = DPAA2_ATOMIC_READ(&sc->buf_num);
+ device_t bpdev;
+ int error;
+
+ /* There's only one buffer pool for now */
+ bpdev = (device_t)rman_get_start(sc->res[DPAA2_NI_BP_RID(0)]);
+ bpsc = device_get_softc(bpdev);
+
+ /* Get state of the buffer pool */
+ error = DPAA2_SWP_QUERY_BP(ch->io_dev, bpsc->attr.bpid, &bpconf);
+ if (error) {
+ device_printf(sc->dev, "%s: DPAA2_SWP_QUERY_BP() failed: "
+ "error=%d\n", __func__, error);
+ return;
+ }
+
+ /* Double allocated Rx buffers if amount of free buffers is < 25% */
+ if (bpconf.free_bufn < (buf_num >> 2)) {
+ mtx_assert(&ch->dma_mtx, MA_NOTOWNED);
+ mtx_lock(&ch->dma_mtx);
+ (void)dpaa2_buf_seed_pool(ch->ni_dev, bpdev, ch, buf_num,
+ DPAA2_RX_BUF_SIZE, &ch->dma_mtx);
+ mtx_unlock(&ch->dma_mtx);
+
+ DPAA2_ATOMIC_XCHG(&sc->buf_free, bpconf.free_bufn);
+ }
+}
diff --git a/sys/dev/dpaa2/dpaa2_io.h b/sys/dev/dpaa2/dpaa2_io.h
--- a/sys/dev/dpaa2/dpaa2_io.h
+++ b/sys/dev/dpaa2/dpaa2_io.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
- * Copyright © 2021-2022 Dmitry Salychev
+ * Copyright © 2021-2023 Dmitry Salychev
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -34,6 +34,7 @@
#include "dpaa2_types.h"
#include "dpaa2_mcp.h"
+#include "dpaa2_swp.h"
/* Maximum resources per DPIO: 3 SYS_MEM + 1 DPMCP. */
#define DPAA2_IO_MAX_RESOURCES 4
@@ -74,8 +75,6 @@
* (CDAN) on a particular WQ channel.
*/
struct dpaa2_io_notif_ctx {
- void (*poll)(void *);
-
device_t io_dev;
void *channel;
uint64_t qman_ctx;
@@ -98,6 +97,9 @@
int irq_rid[DPAA2_IO_MSI_COUNT];
struct resource *irq_resource;
void *intr; /* interrupt handle */
+
+ int cpu;
+ cpuset_t cpu_mask;
};
extern struct resource_spec dpaa2_io_spec[];
diff --git a/sys/dev/dpaa2/dpaa2_io.c b/sys/dev/dpaa2/dpaa2_io.c
--- a/sys/dev/dpaa2/dpaa2_io.c
+++ b/sys/dev/dpaa2/dpaa2_io.c
@@ -38,6 +38,8 @@
* using QBMan.
*/
+#include "opt_rss.h"
+
#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/bus.h>
@@ -58,6 +60,10 @@
#include <dev/pci/pcivar.h>
+#ifdef RSS
+#include <net/rss_config.h>
+#endif
+
#include "pcib_if.h"
#include "pci_if.h"
@@ -68,6 +74,7 @@
#include "dpaa2_cmd_if.h"
#include "dpaa2_io.h"
#include "dpaa2_ni.h"
+#include "dpaa2_channel.h"
#define DPIO_IRQ_INDEX 0 /* index of the only DPIO IRQ */
#define DPIO_POLL_MAX 32
@@ -436,8 +443,14 @@
return (ENXIO);
}
- /* Wrap DPIO ID around number of CPUs. */
- bus_bind_intr(dev, sc->irq_resource, sc->attr.id % mp_ncpus);
+ /* Wrap DPIO ID around number of CPUs/RSS buckets */
+#ifdef RSS
+ sc->cpu = rss_getcpu(sc->attr.id % rss_getnumbuckets());
+#else
+ sc->cpu = sc->attr.id % mp_ncpus;
+#endif
+ CPU_SETOF(sc->cpu, &sc->cpu_mask);
+ bus_bind_intr(dev, sc->irq_resource, sc->cpu);
/*
* Setup and enable Static Dequeue Command to receive CDANs from
@@ -519,7 +532,9 @@
dpaa2_io_intr(void *arg)
{
struct dpaa2_io_softc *sc = (struct dpaa2_io_softc *) arg;
+ /* struct dpaa2_ni_softc *nisc = NULL; */
struct dpaa2_io_notif_ctx *ctx[DPIO_POLL_MAX];
+ struct dpaa2_channel *chan;
struct dpaa2_dq dq;
uint32_t idx, status;
uint16_t flags;
@@ -554,7 +569,9 @@
DPAA2_SWP_UNLOCK(sc->swp);
for (int i = 0; i < cdan_n; i++) {
- ctx[i]->poll(ctx[i]->channel);
+ chan = (struct dpaa2_channel *)ctx[i]->channel;
+ /* nisc = device_get_softc(chan->ni_dev); */
+ taskqueue_enqueue(chan->cleanup_tq, &chan->cleanup_task);
}
/* Enable software portal interrupts back */
diff --git a/sys/dev/dpaa2/dpaa2_mc.c b/sys/dev/dpaa2/dpaa2_mc.c
--- a/sys/dev/dpaa2/dpaa2_mc.c
+++ b/sys/dev/dpaa2/dpaa2_mc.c
@@ -73,8 +73,6 @@
#define mcreg_read_4(_sc, _r) bus_read_4(&(_sc)->map[1], (_r))
#define mcreg_write_4(_sc, _r, _v) bus_write_4(&(_sc)->map[1], (_r), (_v))
-#define COMPARE_TYPE(t, v) (strncmp((v), (t), strlen((v))) == 0)
-
#define IORT_DEVICE_NAME "MCE"
/* MC Registers */
@@ -671,68 +669,6 @@
return (error);
}
-/**
- * @brief Convert DPAA2 device type to string.
- */
-const char *
-dpaa2_ttos(enum dpaa2_dev_type type)
-{
- switch (type) {
- case DPAA2_DEV_MC:
- return ("mc"); /* NOTE: to print as information only. */
- case DPAA2_DEV_RC:
- return ("dprc");
- case DPAA2_DEV_IO:
- return ("dpio");
- case DPAA2_DEV_NI:
- return ("dpni");
- case DPAA2_DEV_MCP:
- return ("dpmcp");
- case DPAA2_DEV_BP:
- return ("dpbp");
- case DPAA2_DEV_CON:
- return ("dpcon");
- case DPAA2_DEV_MAC:
- return ("dpmac");
- case DPAA2_DEV_MUX:
- return ("dpdmux");
- case DPAA2_DEV_SW:
- return ("dpsw");
- default:
- break;
- }
- return ("notype");
-}
-
-/**
- * @brief Convert string to DPAA2 device type.
- */
-enum dpaa2_dev_type
-dpaa2_stot(const char *str)
-{
- if (COMPARE_TYPE(str, "dprc")) {
- return (DPAA2_DEV_RC);
- } else if (COMPARE_TYPE(str, "dpio")) {
- return (DPAA2_DEV_IO);
- } else if (COMPARE_TYPE(str, "dpni")) {
- return (DPAA2_DEV_NI);
- } else if (COMPARE_TYPE(str, "dpmcp")) {
- return (DPAA2_DEV_MCP);
- } else if (COMPARE_TYPE(str, "dpbp")) {
- return (DPAA2_DEV_BP);
- } else if (COMPARE_TYPE(str, "dpcon")) {
- return (DPAA2_DEV_CON);
- } else if (COMPARE_TYPE(str, "dpmac")) {
- return (DPAA2_DEV_MAC);
- } else if (COMPARE_TYPE(str, "dpdmux")) {
- return (DPAA2_DEV_MUX);
- } else if (COMPARE_TYPE(str, "dpsw")) {
- return (DPAA2_DEV_SW);
- }
-
- return (DPAA2_DEV_NOTYPE);
-}
-
/**
* @internal
*/
diff --git a/sys/dev/dpaa2/dpaa2_mcp.h b/sys/dev/dpaa2/dpaa2_mcp.h
--- a/sys/dev/dpaa2/dpaa2_mcp.h
+++ b/sys/dev/dpaa2/dpaa2_mcp.h
@@ -29,7 +29,6 @@
#define _DPAA2_MCP_H
#include <sys/rman.h>
-#include <sys/condvar.h>
#include <sys/mutex.h>
#include "dpaa2_types.h"
diff --git a/sys/dev/dpaa2/dpaa2_mcp.c b/sys/dev/dpaa2/dpaa2_mcp.c
--- a/sys/dev/dpaa2/dpaa2_mcp.c
+++ b/sys/dev/dpaa2/dpaa2_mcp.c
@@ -40,7 +40,6 @@
#include <sys/time.h>
#include <sys/types.h>
#include <sys/systm.h>
-#include <sys/condvar.h>
#include <sys/lock.h>
#include <machine/bus.h>
diff --git a/sys/dev/dpaa2/dpaa2_ni.h b/sys/dev/dpaa2/dpaa2_ni.h
--- a/sys/dev/dpaa2/dpaa2_ni.h
+++ b/sys/dev/dpaa2/dpaa2_ni.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
- * Copyright © 2021-2022 Dmitry Salychev
+ * Copyright © 2021-2023 Dmitry Salychev
* Copyright © 2022 Mathew McBride
*
* Redistribution and use in source and binary forms, with or without
@@ -38,6 +38,7 @@
#include <sys/socket.h>
#include <sys/buf_ring.h>
#include <sys/proc.h>
+#include <sys/lock.h>
#include <sys/mutex.h>
#include <net/if.h>
@@ -50,6 +51,7 @@
#include "dpaa2_io.h"
#include "dpaa2_mac.h"
#include "dpaa2_ni_dpkg.h"
+#include "dpaa2_channel.h"
/* Name of the DPAA2 network interface. */
#define DPAA2_NI_IFNAME "dpni"
@@ -58,17 +60,13 @@
#define DPAA2_NI_MAX_RESOURCES 34
#define DPAA2_NI_MSI_COUNT 1 /* MSIs per DPNI */
-#define DPAA2_NI_MAX_CHANNELS 16 /* to distribute ingress traffic to cores */
-#define DPAA2_NI_MAX_TCS 8 /* traffic classes per DPNI */
#define DPAA2_NI_MAX_POOLS 8 /* buffer pools per DPNI */
-/* Maximum number of Rx buffers. */
-#define DPAA2_NI_BUFS_INIT (50u * DPAA2_SWP_BUFS_PER_CMD)
-#define DPAA2_NI_BUFS_MAX (1 << 15) /* 15 bits for buffer index max. */
+#define DPAA2_NI_BUFS_INIT (5u * DPAA2_SWP_BUFS_PER_CMD)
/* Maximum number of buffers allocated per Tx ring. */
#define DPAA2_NI_BUFS_PER_TX (1 << 7)
-#define DPAA2_NI_MAX_BPTX (1 << 8) /* 8 bits for buffer index max. */
+#define DPAA2_NI_MAX_BPTX (1 << 8)
/* Number of the DPNI statistics counters. */
#define DPAA2_NI_STAT_COUNTERS 7u
@@ -133,12 +131,30 @@
*/
#define DPAA2_NI_ENQUEUE_RETRIES 10
-enum dpaa2_ni_queue_type {
- DPAA2_NI_QUEUE_RX = 0,
- DPAA2_NI_QUEUE_TX,
- DPAA2_NI_QUEUE_TX_CONF,
- DPAA2_NI_QUEUE_RX_ERR
-};
+/* Channel storage buffer configuration */
+#define DPAA2_ETH_STORE_FRAMES 16u
+#define DPAA2_ETH_STORE_SIZE \
+ ((DPAA2_ETH_STORE_FRAMES + 1) * sizeof(struct dpaa2_dq))
+
+/*
+ * NOTE: Don't forget to update dpaa2_ni_spec in case of any changes in macros!
+ */
+/* DPMCP resources */
+#define DPAA2_NI_MCP_RES_NUM (1u)
+#define DPAA2_NI_MCP_RID_OFF (0u)
+#define DPAA2_NI_MCP_RID(rid) ((rid) + DPAA2_NI_MCP_RID_OFF)
+/* DPIO resources (software portals) */
+#define DPAA2_NI_IO_RES_NUM (16u)
+#define DPAA2_NI_IO_RID_OFF (DPAA2_NI_MCP_RID_OFF + DPAA2_NI_MCP_RES_NUM)
+#define DPAA2_NI_IO_RID(rid) ((rid) + DPAA2_NI_IO_RID_OFF)
+/* DPBP resources (buffer pools) */
+#define DPAA2_NI_BP_RES_NUM (1u)
+#define DPAA2_NI_BP_RID_OFF (DPAA2_NI_IO_RID_OFF + DPAA2_NI_IO_RES_NUM)
+#define DPAA2_NI_BP_RID(rid) ((rid) + DPAA2_NI_BP_RID_OFF)
+/* DPCON resources (channels) */
+#define DPAA2_NI_CON_RES_NUM (16u)
+#define DPAA2_NI_CON_RID_OFF (DPAA2_NI_BP_RID_OFF + DPAA2_NI_BP_RES_NUM)
+#define DPAA2_NI_CON_RID(rid) ((rid) + DPAA2_NI_CON_RID_OFF)
enum dpaa2_ni_dest_type {
DPAA2_NI_DEST_NONE = 0,
@@ -172,7 +188,7 @@
DPAA2_NI_ERR_SEND_TO_ERROR_QUEUE
};
-struct dpaa2_ni_channel;
+struct dpaa2_channel;
struct dpaa2_ni_fq;
/**
@@ -203,95 +219,6 @@
} key_size;
};
-/**
- * @brief Tx ring.
- *
- * fq: Parent (TxConf) frame queue.
- * fqid: ID of the logical Tx queue.
- * mbuf_br: Ring buffer for mbufs to transmit.
- * mbuf_lock: Lock for the ring buffer.
- */
-struct dpaa2_ni_tx_ring {
- struct dpaa2_ni_fq *fq;
- uint32_t fqid;
- uint32_t txid; /* Tx ring index */
-
- /* Ring buffer for indexes in "buf" array. */
- struct buf_ring *idx_br;
- struct mtx lock;
-
- /* Buffers to DMA load/unload Tx mbufs. */
- struct dpaa2_buf buf[DPAA2_NI_BUFS_PER_TX];
-};
-
-/**
- * @brief A Frame Queue is the basic queuing structure used by the QMan.
- *
- * It comprises a list of frame descriptors (FDs), so it can be thought of
- * as a queue of frames.
- *
- * NOTE: When frames on a FQ are ready to be processed, the FQ is enqueued
- * onto a work queue (WQ).
- *
- * fqid: Frame queue ID, can be used to enqueue/dequeue or execute other
- * commands on the queue through DPIO.
- * txq_n: Number of configured Tx queues.
- * tx_fqid: Frame queue IDs of the Tx queues which belong to the same flowid.
- * Note that Tx queues are logical queues and not all management
- * commands are available on these queue types.
- * qdbin: Queue destination bin. Can be used with the DPIO enqueue
- * operation based on QDID, QDBIN and QPRI. Note that all Tx queues
- * with the same flowid have the same destination bin.
- */
-struct dpaa2_ni_fq {
- int (*consume)(struct dpaa2_ni_channel *,
- struct dpaa2_ni_fq *, struct dpaa2_fd *);
-
- struct dpaa2_ni_channel *chan;
- uint32_t fqid;
- uint16_t flowid;
- uint8_t tc;
- enum dpaa2_ni_queue_type type;
-
- /* Optional fields (for TxConf queue). */
- struct dpaa2_ni_tx_ring tx_rings[DPAA2_NI_MAX_TCS];
- uint32_t tx_qdbin;
-} __aligned(CACHE_LINE_SIZE);
-
-/**
- * @brief QBMan channel to process ingress traffic (Rx, Tx conf).
- *
- * NOTE: Several WQs are organized into a single WQ Channel.
- */
-struct dpaa2_ni_channel {
- device_t ni_dev;
- device_t io_dev;
- device_t con_dev;
- uint16_t id;
- uint16_t flowid;
-
- /* For debug purposes only! */
- uint64_t tx_frames;
- uint64_t tx_dropped;
-
- /* Context to configure CDAN. */
- struct dpaa2_io_notif_ctx ctx;
-
- /* Channel storage (to keep responses from VDQ command). */
- struct dpaa2_buf store;
- uint32_t store_sz; /* in frames */
- uint32_t store_idx; /* frame index */
-
- /* Recycled buffers to release back to the pool. */
- uint32_t recycled_n;
- struct dpaa2_buf *recycled[DPAA2_SWP_BUFS_PER_CMD];
-
- /* Frame queues */
- uint32_t rxq_n;
- struct dpaa2_ni_fq rx_queues[DPAA2_NI_MAX_TCS];
- struct dpaa2_ni_fq txc_queue;
-};
-
/**
* @brief Configuration of the network interface queue.
*
@@ -534,7 +461,6 @@
uint16_t buf_align;
uint16_t buf_sz;
- /* For debug purposes only! */
uint64_t rx_anomaly_frames;
uint64_t rx_single_buf_frames;
uint64_t rx_sg_buf_frames;
@@ -543,10 +469,8 @@
uint64_t tx_single_buf_frames;
uint64_t tx_sg_frames;
- /* Attributes of the DPAA2 network interface. */
struct dpaa2_ni_attr attr;
- /* For network interface and miibus. */
struct ifnet *ifp;
uint32_t if_flags;
struct mtx lock;
@@ -556,37 +480,25 @@
struct ifmedia fixed_ifmedia;
int media_status;
- /* DMA resources */
- bus_dma_tag_t bp_dmat; /* for buffer pool */
- bus_dma_tag_t tx_dmat; /* for Tx buffers */
- bus_dma_tag_t st_dmat; /* for channel storage */
bus_dma_tag_t rxd_dmat; /* for Rx distribution key */
bus_dma_tag_t qos_dmat; /* for QoS table key */
- bus_dma_tag_t sgt_dmat; /* for scatter/gather tables */
- struct dpaa2_buf qos_kcfg; /* QoS table key config. */
- struct dpaa2_buf rxd_kcfg; /* Rx distribution key config. */
+ struct dpaa2_buf qos_kcfg; /* QoS table key config */
+ struct dpaa2_buf rxd_kcfg; /* Rx distribution key config */
- /* Channels and RxError frame queue */
uint32_t chan_n;
- struct dpaa2_ni_channel *channels[DPAA2_NI_MAX_CHANNELS];
- struct dpaa2_ni_fq rxe_queue; /* one per network interface */
+ struct dpaa2_channel *channels[DPAA2_MAX_CHANNELS];
+ struct dpaa2_ni_fq rxe_queue; /* one per DPNI */
- /* Rx buffers for buffer pool. */
struct dpaa2_atomic buf_num;
struct dpaa2_atomic buf_free; /* for sysctl(9) only */
- struct dpaa2_buf buf[DPAA2_NI_BUFS_MAX];
- /* Interrupts */
int irq_rid[DPAA2_NI_MSI_COUNT];
struct resource *irq_res;
- void *intr; /* interrupt handle */
+ void *intr;
- /* Tasks */
struct taskqueue *bp_taskq;
- struct task bp_task;
- /* Callouts */
struct callout mii_callout;
struct {
@@ -594,7 +506,7 @@
uint8_t addr[ETHER_ADDR_LEN];
device_t phy_dev;
int phy_loc;
- } mac; /* Info about connected DPMAC (if exists). */
+ } mac; /* Info about connected DPMAC (if exists) */
};
extern struct resource_spec dpaa2_ni_spec[];
diff --git a/sys/dev/dpaa2/dpaa2_ni.c b/sys/dev/dpaa2/dpaa2_ni.c
--- a/sys/dev/dpaa2/dpaa2_ni.c
+++ b/sys/dev/dpaa2/dpaa2_ni.c
@@ -94,6 +94,8 @@
#include "dpaa2_swp_if.h"
#include "dpaa2_cmd_if.h"
#include "dpaa2_ni.h"
+#include "dpaa2_channel.h"
+#include "dpaa2_buf.h"
#define BIT(x) (1ul << (x))
#define WRIOP_VERSION(x, y, z) ((x) << 10 | (y) << 5 | (z) << 0)
@@ -115,18 +117,20 @@
mtx_unlock(&(__sc)->lock); \
} while (0)
-#define TX_LOCK(__tx) do { \
- mtx_assert(&(__tx)->lock, MA_NOTOWNED); \
- mtx_lock(&(__tx)->lock); \
-} while (0)
-#define TX_UNLOCK(__tx) do { \
- mtx_assert(&(__tx)->lock, MA_OWNED); \
- mtx_unlock(&(__tx)->lock); \
-} while (0)
-
-#define DPAA2_TX_RING(sc, chan, tc) \
+#define DPAA2_TX_RING(sc, chan, tc) \
(&(sc)->channels[(chan)]->txc_queue.tx_rings[(tc)])
+MALLOC_DEFINE(M_DPAA2_TXB, "dpaa2_txb", "DPAA2 DMA-mapped buffer (Tx)");
+
+/*
+ * How many times channel cleanup routine will be repeated if the RX or TX
+ * budget was depleted.
+ */
+#define DPAA2_CLEAN_BUDGET 64 /* sysctl(9)? */
+/* TX/RX budget for the channel cleanup task */
+#define DPAA2_TX_BUDGET 128 /* sysctl(9)? */
+#define DPAA2_RX_BUDGET 256 /* sysctl(9)? */
+
#define DPNI_IRQ_INDEX 0 /* Index of the only DPNI IRQ. */
#define DPNI_IRQ_LINK_CHANGED 1 /* Link state changed */
#define DPNI_IRQ_EP_CHANGED 2 /* DPAA2 endpoint dis/connected */
@@ -144,24 +148,20 @@
#define BUF_SWA_SIZE 64 /* SW annotation size */
#define BUF_RX_HWA_SIZE 64 /* HW annotation size */
#define BUF_TX_HWA_SIZE 128 /* HW annotation size */
-#define BUF_SIZE (MJUM9BYTES)
-#define DPAA2_TX_BUFRING_SZ (4096u)
+#define DPAA2_RX_BUFRING_SZ (4096u)
+#define DPAA2_RXE_BUFRING_SZ (1024u)
+#define DPAA2_TXC_BUFRING_SZ (4096u)
#define DPAA2_TX_SEGLIMIT (16u) /* arbitrary number */
-#define DPAA2_TX_SEG_SZ (4096u)
+#define DPAA2_TX_SEG_SZ (PAGE_SIZE)
#define DPAA2_TX_SEGS_MAXSZ (DPAA2_TX_SEGLIMIT * DPAA2_TX_SEG_SZ)
#define DPAA2_TX_SGT_SZ (PAGE_SIZE) /* bytes */
/* Size of a buffer to keep a QoS table key configuration. */
-#define ETH_QOS_KCFG_BUF_SIZE 256
+#define ETH_QOS_KCFG_BUF_SIZE (PAGE_SIZE)
/* Required by struct dpni_rx_tc_dist_cfg::key_cfg_iova */
-#define DPAA2_CLASSIFIER_DMA_SIZE 256
-
-/* Channel storage buffer configuration. */
-#define ETH_STORE_FRAMES 16u
-#define ETH_STORE_SIZE ((ETH_STORE_FRAMES + 1) * sizeof(struct dpaa2_dq))
-#define ETH_STORE_ALIGN 64u
+#define DPAA2_CLASSIFIER_DMA_SIZE (PAGE_SIZE)
/* Buffers layout options. */
#define BUF_LOPT_TIMESTAMP 0x1
@@ -221,7 +221,12 @@
MALLOC_DEFINE(M_DPAA2_NI, "dpaa2_ni", "DPAA2 Network Interface");
-/* DPAA2 Network Interface resource specification. */
+/*
+ * DPAA2 Network Interface resource specification.
+ *
+ * NOTE: Don't forget to update macros in dpaa2_ni.h in case of any changes in
+ * the specification!
+ */
struct resource_spec dpaa2_ni_spec[] = {
/*
* DPMCP resources.
@@ -229,11 +234,7 @@
* NOTE: MC command portals (MCPs) are used to send commands to, and
* receive responses from, the MC firmware. One portal per DPNI.
*/
-#define MCP_RES_NUM (1u)
-#define MCP_RID_OFF (0u)
-#define MCP_RID(rid) ((rid) + MCP_RID_OFF)
- /* --- */
- { DPAA2_DEV_MCP, MCP_RID(0), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL },
+ { DPAA2_DEV_MCP, DPAA2_NI_MCP_RID(0), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL },
/*
* DPIO resources (software portals).
*
@@ -241,36 +242,28 @@
* availability interrupts, the DPCONs are used to identify the
* network interface that has produced ingress data to that core.
*/
-#define IO_RES_NUM (16u)
-#define IO_RID_OFF (MCP_RID_OFF + MCP_RES_NUM)
-#define IO_RID(rid) ((rid) + IO_RID_OFF)
- /* --- */
- { DPAA2_DEV_IO, IO_RID(0), RF_ACTIVE | RF_SHAREABLE },
- { DPAA2_DEV_IO, IO_RID(1), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL },
- { DPAA2_DEV_IO, IO_RID(2), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL },
- { DPAA2_DEV_IO, IO_RID(3), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL },
- { DPAA2_DEV_IO, IO_RID(4), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL },
- { DPAA2_DEV_IO, IO_RID(5), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL },
- { DPAA2_DEV_IO, IO_RID(6), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL },
- { DPAA2_DEV_IO, IO_RID(7), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL },
- { DPAA2_DEV_IO, IO_RID(8), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL },
- { DPAA2_DEV_IO, IO_RID(9), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL },
- { DPAA2_DEV_IO, IO_RID(10), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL },
- { DPAA2_DEV_IO, IO_RID(11), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL },
- { DPAA2_DEV_IO, IO_RID(12), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL },
- { DPAA2_DEV_IO, IO_RID(13), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL },
- { DPAA2_DEV_IO, IO_RID(14), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL },
- { DPAA2_DEV_IO, IO_RID(15), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL },
+ { DPAA2_DEV_IO, DPAA2_NI_IO_RID(0), RF_ACTIVE | RF_SHAREABLE },
+ { DPAA2_DEV_IO, DPAA2_NI_IO_RID(1), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL },
+ { DPAA2_DEV_IO, DPAA2_NI_IO_RID(2), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL },
+ { DPAA2_DEV_IO, DPAA2_NI_IO_RID(3), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL },
+ { DPAA2_DEV_IO, DPAA2_NI_IO_RID(4), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL },
+ { DPAA2_DEV_IO, DPAA2_NI_IO_RID(5), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL },
+ { DPAA2_DEV_IO, DPAA2_NI_IO_RID(6), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL },
+ { DPAA2_DEV_IO, DPAA2_NI_IO_RID(7), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL },
+ { DPAA2_DEV_IO, DPAA2_NI_IO_RID(8), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL },
+ { DPAA2_DEV_IO, DPAA2_NI_IO_RID(9), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL },
+ { DPAA2_DEV_IO, DPAA2_NI_IO_RID(10), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL },
+ { DPAA2_DEV_IO, DPAA2_NI_IO_RID(11), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL },
+ { DPAA2_DEV_IO, DPAA2_NI_IO_RID(12), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL },
+ { DPAA2_DEV_IO, DPAA2_NI_IO_RID(13), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL },
+ { DPAA2_DEV_IO, DPAA2_NI_IO_RID(14), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL },
+ { DPAA2_DEV_IO, DPAA2_NI_IO_RID(15), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL },
/*
* DPBP resources (buffer pools).
*
* NOTE: One per network interface.
*/
-#define BP_RES_NUM (1u)
-#define BP_RID_OFF (IO_RID_OFF + IO_RES_NUM)
-#define BP_RID(rid) ((rid) + BP_RID_OFF)
- /* --- */
- { DPAA2_DEV_BP, BP_RID(0), RF_ACTIVE },
+ { DPAA2_DEV_BP, DPAA2_NI_BP_RID(0), RF_ACTIVE },
/*
* DPCON resources (channels).
*
@@ -280,27 +273,23 @@
* different network interfaces arriving on the same core, the
* DPCONs must be private to the DPNIs.
*/
-#define CON_RES_NUM (16u)
-#define CON_RID_OFF (BP_RID_OFF + BP_RES_NUM)
-#define CON_RID(rid) ((rid) + CON_RID_OFF)
- /* --- */
- { DPAA2_DEV_CON, CON_RID(0), RF_ACTIVE },
- { DPAA2_DEV_CON, CON_RID(1), RF_ACTIVE | RF_OPTIONAL },
- { DPAA2_DEV_CON, CON_RID(2), RF_ACTIVE | RF_OPTIONAL },
- { DPAA2_DEV_CON, CON_RID(3), RF_ACTIVE | RF_OPTIONAL },
- { DPAA2_DEV_CON, CON_RID(4), RF_ACTIVE | RF_OPTIONAL },
- { DPAA2_DEV_CON, CON_RID(5), RF_ACTIVE | RF_OPTIONAL },
- { DPAA2_DEV_CON, CON_RID(6), RF_ACTIVE | RF_OPTIONAL },
- { DPAA2_DEV_CON, CON_RID(7), RF_ACTIVE | RF_OPTIONAL },
- { DPAA2_DEV_CON, CON_RID(8), RF_ACTIVE | RF_OPTIONAL },
- { DPAA2_DEV_CON, CON_RID(9), RF_ACTIVE | RF_OPTIONAL },
- { DPAA2_DEV_CON, CON_RID(10), RF_ACTIVE | RF_OPTIONAL },
- { DPAA2_DEV_CON, CON_RID(11), RF_ACTIVE | RF_OPTIONAL },
- { DPAA2_DEV_CON, CON_RID(12), RF_ACTIVE | RF_OPTIONAL },
- { DPAA2_DEV_CON, CON_RID(13), RF_ACTIVE | RF_OPTIONAL },
- { DPAA2_DEV_CON, CON_RID(14), RF_ACTIVE | RF_OPTIONAL },
- { DPAA2_DEV_CON, CON_RID(15), RF_ACTIVE | RF_OPTIONAL },
- /* --- */
+ { DPAA2_DEV_CON, DPAA2_NI_CON_RID(0), RF_ACTIVE },
+ { DPAA2_DEV_CON, DPAA2_NI_CON_RID(1), RF_ACTIVE | RF_OPTIONAL },
+ { DPAA2_DEV_CON, DPAA2_NI_CON_RID(2), RF_ACTIVE | RF_OPTIONAL },
+ { DPAA2_DEV_CON, DPAA2_NI_CON_RID(3), RF_ACTIVE | RF_OPTIONAL },
+ { DPAA2_DEV_CON, DPAA2_NI_CON_RID(4), RF_ACTIVE | RF_OPTIONAL },
+ { DPAA2_DEV_CON, DPAA2_NI_CON_RID(5), RF_ACTIVE | RF_OPTIONAL },
+ { DPAA2_DEV_CON, DPAA2_NI_CON_RID(6), RF_ACTIVE | RF_OPTIONAL },
+ { DPAA2_DEV_CON, DPAA2_NI_CON_RID(7), RF_ACTIVE | RF_OPTIONAL },
+ { DPAA2_DEV_CON, DPAA2_NI_CON_RID(8), RF_ACTIVE | RF_OPTIONAL },
+ { DPAA2_DEV_CON, DPAA2_NI_CON_RID(9), RF_ACTIVE | RF_OPTIONAL },
+ { DPAA2_DEV_CON, DPAA2_NI_CON_RID(10), RF_ACTIVE | RF_OPTIONAL },
+ { DPAA2_DEV_CON, DPAA2_NI_CON_RID(11), RF_ACTIVE | RF_OPTIONAL },
+ { DPAA2_DEV_CON, DPAA2_NI_CON_RID(12), RF_ACTIVE | RF_OPTIONAL },
+ { DPAA2_DEV_CON, DPAA2_NI_CON_RID(13), RF_ACTIVE | RF_OPTIONAL },
+ { DPAA2_DEV_CON, DPAA2_NI_CON_RID(14), RF_ACTIVE | RF_OPTIONAL },
+ { DPAA2_DEV_CON, DPAA2_NI_CON_RID(15), RF_ACTIVE | RF_OPTIONAL },
+
RESOURCE_SPEC_END
};
@@ -391,6 +380,13 @@
"depletion in DPNI buffer pools" },
};
+struct dpaa2_ni_rx_ctx {
+ struct mbuf *head;
+ struct mbuf *tail;
+ int cnt;
+ bool last;
+};
+
/* Device interface */
static int dpaa2_ni_probe(device_t);
static int dpaa2_ni_attach(device_t);
@@ -399,8 +395,6 @@
/* DPAA2 network interface setup and configuration */
static int dpaa2_ni_setup(device_t);
static int dpaa2_ni_setup_channels(device_t);
-static int dpaa2_ni_setup_fq(device_t, struct dpaa2_ni_channel *,
- enum dpaa2_ni_queue_type);
static int dpaa2_ni_bind(device_t);
static int dpaa2_ni_setup_rx_dist(device_t);
static int dpaa2_ni_setup_irqs(device_t);
@@ -423,13 +417,6 @@
static int dpaa2_ni_set_hash(device_t, uint64_t);
static int dpaa2_ni_set_dist_key(device_t, enum dpaa2_ni_dist_mode, uint64_t);
-/* Buffers and buffer pools */
-static int dpaa2_ni_seed_buf_pool(struct dpaa2_ni_softc *, uint32_t);
-static int dpaa2_ni_seed_rxbuf(struct dpaa2_ni_softc *, struct dpaa2_buf *);
-static int dpaa2_ni_seed_txbuf(struct dpaa2_ni_softc *, struct dpaa2_buf *);
-static int dpaa2_ni_seed_chan_storage(struct dpaa2_ni_softc *,
- struct dpaa2_ni_channel *);
-
/* Frame descriptor routines */
static int dpaa2_ni_build_fd(struct dpaa2_ni_softc *, struct dpaa2_ni_tx_ring *,
struct dpaa2_buf *, bus_dma_segment_t *, int, struct dpaa2_fd *);
@@ -442,8 +429,6 @@
/* Various subroutines */
static int dpaa2_ni_cmp_api_version(struct dpaa2_ni_softc *, uint16_t, uint16_t);
static int dpaa2_ni_prepare_key_cfg(struct dpkg_profile_cfg *, uint8_t *);
-static int dpaa2_ni_chan_storage_next(struct dpaa2_ni_channel *,
- struct dpaa2_dq **);
/* Network interface routines */
static void dpaa2_ni_init(void *);
@@ -462,23 +447,21 @@
static void dpaa2_ni_media_status(if_t , struct ifmediareq *);
static void dpaa2_ni_media_tick(void *);
-/* DMA mapping callback */
-static void dpaa2_ni_dmamap_cb(void *, bus_dma_segment_t *, int, int);
-
/* Tx/Rx routines. */
-static void dpaa2_ni_poll(void *);
-static void dpaa2_ni_tx_locked(struct dpaa2_ni_softc *,
+static int dpaa2_ni_rx_cleanup(struct dpaa2_channel *);
+static int dpaa2_ni_tx_cleanup(struct dpaa2_channel *);
+static void dpaa2_ni_tx(struct dpaa2_ni_softc *, struct dpaa2_channel *,
struct dpaa2_ni_tx_ring *, struct mbuf *);
-static void dpaa2_ni_bp_task(void *, int);
+static void dpaa2_ni_cleanup_task(void *, int);
/* Tx/Rx subroutines */
-static int dpaa2_ni_consume_frames(struct dpaa2_ni_channel *,
- struct dpaa2_ni_fq **, uint32_t *);
-static int dpaa2_ni_rx(struct dpaa2_ni_channel *, struct dpaa2_ni_fq *,
- struct dpaa2_fd *);
-static int dpaa2_ni_rx_err(struct dpaa2_ni_channel *, struct dpaa2_ni_fq *,
+static int dpaa2_ni_consume_frames(struct dpaa2_channel *, struct dpaa2_ni_fq **,
+ uint32_t *);
+static int dpaa2_ni_rx(struct dpaa2_channel *, struct dpaa2_ni_fq *,
+ struct dpaa2_fd *, struct dpaa2_ni_rx_ctx *);
+static int dpaa2_ni_rx_err(struct dpaa2_channel *, struct dpaa2_ni_fq *,
struct dpaa2_fd *);
-static int dpaa2_ni_tx_conf(struct dpaa2_ni_channel *, struct dpaa2_ni_fq *,
+static int dpaa2_ni_tx_conf(struct dpaa2_channel *, struct dpaa2_ni_fq *,
struct dpaa2_fd *);
/* sysctl(9) */
@@ -531,20 +514,16 @@
DPAA2_ATOMIC_XCHG(&sc->buf_num, 0);
DPAA2_ATOMIC_XCHG(&sc->buf_free, 0);
- sc->bp_dmat = NULL;
- sc->st_dmat = NULL;
sc->rxd_dmat = NULL;
sc->qos_dmat = NULL;
- sc->qos_kcfg.type = DPAA2_BUF_STORE;
- sc->qos_kcfg.store.dmap = NULL;
- sc->qos_kcfg.store.paddr = 0;
- sc->qos_kcfg.store.vaddr = NULL;
+ sc->qos_kcfg.dmap = NULL;
+ sc->qos_kcfg.paddr = 0;
+ sc->qos_kcfg.vaddr = NULL;
- sc->rxd_kcfg.type = DPAA2_BUF_STORE;
- sc->rxd_kcfg.store.dmap = NULL;
- sc->rxd_kcfg.store.paddr = 0;
- sc->rxd_kcfg.store.vaddr = NULL;
+ sc->rxd_kcfg.dmap = NULL;
+ sc->rxd_kcfg.paddr = 0;
+ sc->rxd_kcfg.vaddr = NULL;
sc->mac.dpmac_id = 0;
sc->mac.phy_dev = NULL;
@@ -558,7 +537,7 @@
}
/* Obtain MC portal. */
- mcp_dev = (device_t) rman_get_start(sc->res[MCP_RID(0)]);
+ mcp_dev = (device_t) rman_get_start(sc->res[DPAA2_NI_MCP_RID(0)]);
mcp_dinfo = device_get_ivars(mcp_dev);
dinfo->portal = mcp_dinfo->portal;
@@ -600,16 +579,15 @@
goto close_rc;
}
+ bzero(tq_name, sizeof(tq_name));
+ snprintf(tq_name, sizeof(tq_name), "%s_tqbp", device_get_nameunit(dev));
+
/*
* XXX-DSL: Release new buffers on Buffer Pool State Change Notification
* (BPSCN) returned as a result to the VDQ command instead.
* It is similar to CDAN processed in dpaa2_io_intr().
*/
/* Create a taskqueue thread to release new buffers to the pool. */
- TASK_INIT(&sc->bp_task, 0, dpaa2_ni_bp_task, sc);
- bzero(tq_name, sizeof (tq_name));
- snprintf(tq_name, sizeof (tq_name), "%s_tqbp",
- device_get_nameunit(dev));
sc->bp_taskq = taskqueue_create(tq_name, M_WAITOK,
taskqueue_thread_enqueue, &sc->bp_taskq);
if (sc->bp_taskq == NULL) {
@@ -619,6 +597,11 @@
}
taskqueue_start_threads(&sc->bp_taskq, 1, PI_NET, "%s", tq_name);
+ /* sc->cleanup_taskq = taskqueue_create("dpaa2_ch cleanup", M_WAITOK, */
+ /* taskqueue_thread_enqueue, &sc->cleanup_taskq); */
+ /* taskqueue_start_threads(&sc->cleanup_taskq, 1, PI_NET, */
+ /* "dpaa2_ch cleanup"); */
+
error = dpaa2_ni_setup(dev);
if (error) {
device_printf(dev, "%s: failed to setup DPNI: error=%d\n",
@@ -999,212 +982,52 @@
static int
dpaa2_ni_setup_channels(device_t dev)
{
- device_t pdev = device_get_parent(dev);
- device_t child = dev;
- device_t io_dev, con_dev;
+ device_t iodev, condev, bpdev;
struct dpaa2_ni_softc *sc = device_get_softc(dev);
- struct dpaa2_ni_channel *channel;
- struct dpaa2_con_softc *consc;
- struct dpaa2_con_notif_cfg notif_cfg;
- struct dpaa2_devinfo *rc_info = device_get_ivars(pdev);
- struct dpaa2_devinfo *io_info;
- struct dpaa2_devinfo *con_info;
- struct dpaa2_io_notif_ctx *ctx;
- struct dpaa2_buf *buf;
- struct dpaa2_cmd cmd;
- struct sysctl_ctx_list *sysctl_ctx;
- struct sysctl_oid *node;
- struct sysctl_oid_list *parent;
uint32_t i, num_chan;
- uint16_t rc_token, con_token;
int error;
- /* Calculate number of the channels based on the allocated resources. */
- for (i = 0; i < IO_RES_NUM; i++) {
- if (!sc->res[IO_RID(i)]) {
+ /* Calculate number of the channels based on the allocated resources */
+ for (i = 0; i < DPAA2_NI_IO_RES_NUM; i++) {
+ if (!sc->res[DPAA2_NI_IO_RID(i)]) {
break;
}
}
num_chan = i;
- for (i = 0; i < CON_RES_NUM; i++) {
- if (!sc->res[CON_RID(i)]) {
+ for (i = 0; i < DPAA2_NI_CON_RES_NUM; i++) {
+ if (!sc->res[DPAA2_NI_CON_RID(i)]) {
break;
}
}
num_chan = i < num_chan ? i : num_chan;
- sc->chan_n = num_chan > DPAA2_NI_MAX_CHANNELS
- ? DPAA2_NI_MAX_CHANNELS : num_chan;
+ sc->chan_n = num_chan > DPAA2_MAX_CHANNELS
+ ? DPAA2_MAX_CHANNELS : num_chan;
sc->chan_n = sc->chan_n > sc->attr.num.queues
? sc->attr.num.queues : sc->chan_n;
- device_printf(dev, "channels=%d\n", sc->chan_n);
-
- sysctl_ctx = device_get_sysctl_ctx(sc->dev);
- parent = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev));
- node = SYSCTL_ADD_NODE(sysctl_ctx, parent, OID_AUTO, "channels",
- CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "DPNI Channels");
- parent = SYSCTL_CHILDREN(node);
-
- /* Setup channels for the portal. */
- for (uint32_t i = 0; i < sc->chan_n; i++) {
- io_dev = (device_t) rman_get_start(sc->res[IO_RID(i)]);
- io_info = device_get_ivars(io_dev);
-
- con_dev = (device_t) rman_get_start(sc->res[CON_RID(i)]);
- consc = device_get_softc(con_dev);
- con_info = device_get_ivars(con_dev);
-
- DPAA2_CMD_INIT(&cmd);
+ KASSERT(sc->chan_n > 0u, ("%s: positive number of channels expected: "
+ "chan_n=%d", __func__, sc->chan_n));
- error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rc_info->id,
- &rc_token);
- if (error) {
- device_printf(dev, "%s: failed to open resource "
- "container: id=%d, error=%d\n", __func__,
- rc_info->id, error);
- return (error);
- }
- error = DPAA2_CMD_CON_OPEN(dev, child, &cmd, con_info->id,
- &con_token);
- if (error) {
- device_printf(dev, "%s: failed to open DPCON: id=%d, "
- "error=%d\n", __func__, con_info->id, error);
- (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd,
- rc_token));
- return (error);
- }
-
- error = DPAA2_CMD_CON_ENABLE(dev, child, &cmd);
- if (error) {
- device_printf(dev, "%s: failed to enable channel: "
- "dpcon_id=%d, chan_id=%d\n", __func__, con_info->id,
- consc->attr.chan_id);
- (void)DPAA2_CMD_CON_CLOSE(dev, child, &cmd);
- (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd,
- rc_token));
- return (error);
- }
-
- channel = malloc(sizeof(struct dpaa2_ni_channel), M_DPAA2_NI,
- M_WAITOK | M_ZERO);
- if (!channel) {
- device_printf(dev, "%s: failed to allocate a channel\n",
- __func__);
- (void)DPAA2_CMD_CON_CLOSE(dev, child, &cmd);
- (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd,
- rc_token));
- return (ENOMEM);
- }
-
- sc->channels[i] = channel;
-
- channel->id = consc->attr.chan_id;
- channel->flowid = i;
- channel->ni_dev = dev;
- channel->io_dev = io_dev;
- channel->con_dev = con_dev;
- channel->recycled_n = 0;
- channel->tx_frames = 0; /* for debug purposes */
- channel->tx_dropped = 0; /* for debug purposes */
- channel->rxq_n = 0;
-
- buf = &channel->store;
- buf->type = DPAA2_BUF_STORE;
- buf->store.dmat = NULL;
- buf->store.dmap = NULL;
- buf->store.paddr = 0;
- buf->store.vaddr = NULL;
-
- /* Setup WQ channel notification context. */
- ctx = &channel->ctx;
- ctx->qman_ctx = (uint64_t) ctx;
- ctx->cdan_en = true;
- ctx->fq_chan_id = channel->id;
- ctx->io_dev = channel->io_dev;
- ctx->channel = channel;
- ctx->poll = dpaa2_ni_poll;
-
- /* Register the new notification context. */
- error = DPAA2_SWP_CONF_WQ_CHANNEL(channel->io_dev, ctx);
- if (error) {
- device_printf(dev, "%s: failed to register notification "
- "context\n", __func__);
- (void)DPAA2_CMD_CON_CLOSE(dev, child, &cmd);
- (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd,
- rc_token));
- return (error);
- }
-
- /* Register DPCON notification with Management Complex. */
- notif_cfg.dpio_id = io_info->id;
- notif_cfg.prior = 0;
- notif_cfg.qman_ctx = ctx->qman_ctx;
- error = DPAA2_CMD_CON_SET_NOTIF(dev, child, &cmd, &notif_cfg);
- if (error) {
- device_printf(dev, "%s: failed to set DPCON "
- "notification: dpcon_id=%d, chan_id=%d\n", __func__,
- con_info->id, consc->attr.chan_id);
- (void)DPAA2_CMD_CON_CLOSE(dev, child, &cmd);
- (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd,
- rc_token));
- return (error);
- }
+ device_printf(dev, "channels=%d\n", sc->chan_n);
- /* Allocate initial # of Rx buffers and a channel storage. */
- error = dpaa2_ni_seed_buf_pool(sc, DPAA2_NI_BUFS_INIT);
- if (error) {
- device_printf(dev, "%s: failed to seed buffer pool\n",
- __func__);
- (void)DPAA2_CMD_CON_CLOSE(dev, child, &cmd);
- (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd,
- rc_token));
- return (error);
- }
- error = dpaa2_ni_seed_chan_storage(sc, channel);
- if (error) {
- device_printf(dev, "%s: failed to seed channel "
- "storage\n", __func__);
- (void)DPAA2_CMD_CON_CLOSE(dev, child, &cmd);
- (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd,
- rc_token));
- return (error);
- }
+ for (i = 0; i < sc->chan_n; i++) {
+ iodev = (device_t)rman_get_start(sc->res[DPAA2_NI_IO_RID(i)]);
+ condev = (device_t)rman_get_start(sc->res[DPAA2_NI_CON_RID(i)]);
+ /* Only one buffer pool available at the moment */
+ bpdev = (device_t)rman_get_start(sc->res[DPAA2_NI_BP_RID(0)]);
- /* Prepare queues for this channel. */
- error = dpaa2_ni_setup_fq(dev, channel, DPAA2_NI_QUEUE_TX_CONF);
- if (error) {
- device_printf(dev, "%s: failed to prepare TxConf "
- "queue: error=%d\n", __func__, error);
- (void)DPAA2_CMD_CON_CLOSE(dev, child, &cmd);
- (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd,
- rc_token));
- return (error);
- }
- error = dpaa2_ni_setup_fq(dev, channel, DPAA2_NI_QUEUE_RX);
- if (error) {
- device_printf(dev, "%s: failed to prepare Rx queue: "
- "error=%d\n", __func__, error);
- (void)DPAA2_CMD_CON_CLOSE(dev, child, &cmd);
- (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd,
- rc_token));
+ error = dpaa2_chan_setup(dev, iodev, condev, bpdev,
+ &sc->channels[i], i, dpaa2_ni_cleanup_task);
+ if (error != 0) {
+ device_printf(dev, "%s: dpaa2_chan_setup() failed: "
+ "error=%d, chan_id=%d\n", __func__, error, i);
return (error);
}
-
- if (bootverbose) {
- device_printf(dev, "channel: dpio_id=%d "
- "dpcon_id=%d chan_id=%d, priorities=%d\n",
- io_info->id, con_info->id, channel->id,
- consc->attr.prior_num);
- }
-
- (void)DPAA2_CMD_CON_CLOSE(dev, child, &cmd);
- (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd,
- rc_token));
}
- /* There is exactly one Rx error queue per DPNI. */
- error = dpaa2_ni_setup_fq(dev, sc->channels[0], DPAA2_NI_QUEUE_RX_ERR);
- if (error) {
+ /* There is exactly one Rx error queue per network interface */
+ error = dpaa2_chan_setup_fq(dev, sc->channels[0], DPAA2_NI_QUEUE_RX_ERR);
+ if (error != 0) {
device_printf(dev, "%s: failed to prepare RxError queue: "
"error=%d\n", __func__, error);
return (error);
@@ -1213,65 +1036,6 @@
return (0);
}
-/**
- * @brief Performs an initial configuration of the frame queues.
- */
-static int
-dpaa2_ni_setup_fq(device_t dev, struct dpaa2_ni_channel *chan,
- enum dpaa2_ni_queue_type queue_type)
-{
- struct dpaa2_ni_softc *sc = device_get_softc(dev);
- struct dpaa2_ni_fq *fq;
-
- switch (queue_type) {
- case DPAA2_NI_QUEUE_TX_CONF:
- /* One queue per channel. */
- fq = &chan->txc_queue;
-
- fq->consume = dpaa2_ni_tx_conf;
- fq->chan = chan;
- fq->flowid = chan->flowid;
- fq->tc = 0; /* ignored */
- fq->type = queue_type;
-
- break;
- case DPAA2_NI_QUEUE_RX:
- KASSERT(sc->attr.num.rx_tcs <= DPAA2_NI_MAX_TCS,
- ("too many Rx traffic classes: rx_tcs=%d\n",
- sc->attr.num.rx_tcs));
-
- /* One queue per Rx traffic class within a channel. */
- for (int i = 0; i < sc->attr.num.rx_tcs; i++) {
- fq = &chan->rx_queues[i];
-
- fq->consume = dpaa2_ni_rx;
- fq->chan = chan;
- fq->flowid = chan->flowid;
- fq->tc = (uint8_t) i;
- fq->type = queue_type;
-
- chan->rxq_n++;
- }
- break;
- case DPAA2_NI_QUEUE_RX_ERR:
- /* One queue per network interface. */
- fq = &sc->rxe_queue;
-
- fq->consume = dpaa2_ni_rx_err;
- fq->chan = chan;
- fq->flowid = 0; /* ignored */
- fq->tc = 0; /* ignored */
- fq->type = queue_type;
- break;
- default:
- device_printf(dev, "%s: unexpected frame queue type: %d\n",
- __func__, queue_type);
- return (EINVAL);
- }
-
- return (0);
-}
-
/**
* @brief Bind DPNI to DPBPs, DPIOs, frame queues and channels.
*/
@@ -1288,7 +1052,7 @@
struct dpaa2_cmd cmd;
struct dpaa2_ni_pools_cfg pools_cfg;
struct dpaa2_ni_err_cfg err_cfg;
- struct dpaa2_ni_channel *chan;
+ struct dpaa2_channel *chan;
uint16_t rc_token, ni_token;
int error;
@@ -1308,7 +1072,7 @@
}
/* Select buffer pool (only one available at the moment). */
- bp_dev = (device_t) rman_get_start(sc->res[BP_RID(0)]);
+ bp_dev = (device_t) rman_get_start(sc->res[DPAA2_NI_BP_RID(0)]);
bp_info = device_get_ivars(bp_dev);
/* Configure buffers pool. */
@@ -1499,6 +1263,7 @@
device_t pdev = device_get_parent(dev);
device_t child = dev;
struct dpaa2_ni_softc *sc = device_get_softc(dev);
+ struct dpaa2_channel *ch = fq->chan;
struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev);
struct dpaa2_devinfo *dinfo = device_get_ivars(dev);
struct dpaa2_devinfo *con_info;
@@ -1528,7 +1293,7 @@
/* Obtain DPCON associated with the FQ's channel. */
con_info = device_get_ivars(fq->chan->con_dev);
- KASSERT(sc->attr.num.tx_tcs <= DPAA2_NI_MAX_TCS,
+ KASSERT(sc->attr.num.tx_tcs <= DPAA2_MAX_TCS,
("%s: too many Tx traffic classes: tx_tcs=%d\n", __func__,
sc->attr.num.tx_tcs));
KASSERT(DPAA2_NI_BUFS_PER_TX <= DPAA2_NI_MAX_BPTX,
@@ -1564,29 +1329,40 @@
mtx_init(&tx->lock, "dpaa2_tx_ring", NULL, MTX_DEF);
/* Allocate Tx ring buffer. */
- tx->idx_br = buf_ring_alloc(DPAA2_TX_BUFRING_SZ, M_DEVBUF,
- M_NOWAIT, &tx->lock);
- if (tx->idx_br == NULL) {
+ tx->br = buf_ring_alloc(DPAA2_TX_BUFRING_SZ, M_DEVBUF, M_NOWAIT,
+ &tx->lock);
+ if (tx->br == NULL) {
device_printf(dev, "%s: failed to setup Tx ring buffer"
" (2) fqid=%d\n", __func__, tx->fqid);
goto close_ni;
}
- /* Configure Tx buffers. */
+ /* Configure Tx buffers */
for (uint64_t j = 0; j < DPAA2_NI_BUFS_PER_TX; j++) {
- buf = &tx->buf[j];
- buf->type = DPAA2_BUF_TX;
- buf->tx.dmat = buf->tx.sgt_dmat = NULL;
- buf->tx.dmap = buf->tx.sgt_dmap = NULL;
- buf->tx.paddr = buf->tx.sgt_paddr = 0;
- buf->tx.vaddr = buf->tx.sgt_vaddr = NULL;
- buf->tx.m = NULL;
- buf->tx.idx = j;
-
- error = dpaa2_ni_seed_txbuf(sc, buf);
-
- /* Add index of the Tx buffer to the ring. */
- buf_ring_enqueue(tx->idx_br, (void *) j);
+ buf = malloc(sizeof(struct dpaa2_buf), M_DPAA2_TXB,
+ M_WAITOK);
+ if (buf == NULL) {
+ device_printf(dev, "%s: malloc() failed (buf)\n",
+ __func__);
+ return (ENOMEM);
+ }
+ /* Keep DMA tag and Tx ring linked to the buffer */
+ DPAA2_BUF_INIT_TAGOPT(buf, ch->tx_dmat, tx);
+
+ buf->sgt = malloc(sizeof(struct dpaa2_buf), M_DPAA2_TXB,
+ M_WAITOK);
+ if (buf->sgt == NULL) {
+ device_printf(dev, "%s: malloc() failed (sgt)\n",
+ __func__);
+ return (ENOMEM);
+ }
+ /* Link SGT to DMA tag and back to its Tx buffer */
+ DPAA2_BUF_INIT_TAGOPT(buf->sgt, ch->sgt_dmat, buf);
+
+ error = dpaa2_buf_seed_txb(dev, buf);
+
+ /* Add Tx buffer to the ring */
+ buf_ring_enqueue(tx->br, buf);
}
tx_rings_n++;
@@ -2021,58 +1797,6 @@
KASSERT((sc->buf_align == BUF_ALIGN) || (sc->buf_align == BUF_ALIGN_V1),
("unexpected buffer alignment: %d\n", sc->buf_align));
- /* DMA tag to allocate buffers for Rx buffer pool. */
- error = bus_dma_tag_create(
- bus_get_dma_tag(dev),
- sc->buf_align, 0, /* alignment, boundary */
- BUS_SPACE_MAXADDR, /* low restricted addr */
- BUS_SPACE_MAXADDR, /* high restricted addr */
- NULL, NULL, /* filter, filterarg */
- BUF_SIZE, 1, /* maxsize, nsegments */
- BUF_SIZE, 0, /* maxsegsize, flags */
- NULL, NULL, /* lockfunc, lockarg */
- &sc->bp_dmat);
- if (error) {
- device_printf(dev, "%s: failed to create DMA tag for buffer "
- "pool\n", __func__);
- return (error);
- }
-
- /* DMA tag to map Tx mbufs. */
- error = bus_dma_tag_create(
- bus_get_dma_tag(dev),
- sc->buf_align, 0, /* alignment, boundary */
- BUS_SPACE_MAXADDR, /* low restricted addr */
- BUS_SPACE_MAXADDR, /* high restricted addr */
- NULL, NULL, /* filter, filterarg */
- DPAA2_TX_SEGS_MAXSZ, /* maxsize */
- DPAA2_TX_SEGLIMIT, /* nsegments */
- DPAA2_TX_SEG_SZ, 0, /* maxsegsize, flags */
- NULL, NULL, /* lockfunc, lockarg */
- &sc->tx_dmat);
- if (error) {
- device_printf(dev, "%s: failed to create DMA tag for Tx "
- "buffers\n", __func__);
- return (error);
- }
-
- /* DMA tag to allocate channel storage. */
- error = bus_dma_tag_create(
- bus_get_dma_tag(dev),
- ETH_STORE_ALIGN, 0, /* alignment, boundary */
- BUS_SPACE_MAXADDR, /* low restricted addr */
- BUS_SPACE_MAXADDR, /* high restricted addr */
- NULL, NULL, /* filter, filterarg */
- ETH_STORE_SIZE, 1, /* maxsize, nsegments */
- ETH_STORE_SIZE, 0, /* maxsegsize, flags */
- NULL, NULL, /* lockfunc, lockarg */
- &sc->st_dmat);
- if (error) {
- device_printf(dev, "%s: failed to create DMA tag for channel "
- "storage\n", __func__);
- return (error);
- }
-
/* DMA tag for Rx distribution key. */
error = bus_dma_tag_create(
bus_get_dma_tag(dev),
@@ -2106,22 +1830,6 @@
return (error);
}
- error = bus_dma_tag_create(
- bus_get_dma_tag(dev),
- PAGE_SIZE, 0, /* alignment, boundary */
- BUS_SPACE_MAXADDR, /* low restricted addr */
- BUS_SPACE_MAXADDR, /* high restricted addr */
- NULL, NULL, /* filter, filterarg */
- DPAA2_TX_SGT_SZ, 1, /* maxsize, nsegments */
- DPAA2_TX_SGT_SZ, 0, /* maxsegsize, flags */
- NULL, NULL, /* lockfunc, lockarg */
- &sc->sgt_dmat);
- if (error) {
- device_printf(dev, "%s: failed to create DMA tag for S/G "
- "tables\n", __func__);
- return (error);
- }
-
return (0);
}
@@ -2169,7 +1877,7 @@
* We need to ensure that the buffer size seen by WRIOP is a multiple
* of 64 or 256 bytes depending on the WRIOP version.
*/
- sc->buf_sz = ALIGN_DOWN(BUF_SIZE, sc->buf_align);
+ sc->buf_sz = ALIGN_DOWN(DPAA2_RX_BUF_SIZE, sc->buf_align);
if (bootverbose) {
device_printf(dev, "Rx/Tx buffers: size=%d, alignment=%d\n",
@@ -2389,23 +2097,22 @@
* Allocate a buffer visible to the device to hold the QoS table key
* configuration.
*/
- KASSERT(buf->type == DPAA2_BUF_STORE, ("%s: not storage buffer",
- __func__));
- if (__predict_true(buf->store.dmat == NULL)) {
- buf->store.dmat = sc->qos_dmat;
+
+ if (__predict_true(buf->dmat == NULL)) {
+ buf->dmat = sc->qos_dmat;
}
- error = bus_dmamem_alloc(buf->store.dmat, &buf->store.vaddr,
- BUS_DMA_ZERO | BUS_DMA_COHERENT, &buf->store.dmap);
+ error = bus_dmamem_alloc(buf->dmat, (void **)&buf->vaddr,
+ BUS_DMA_ZERO | BUS_DMA_COHERENT, &buf->dmap);
if (error) {
device_printf(dev, "%s: failed to allocate a buffer for QoS key "
"configuration\n", __func__);
goto err_exit;
}
- error = bus_dmamap_load(buf->store.dmat, buf->store.dmap,
- buf->store.vaddr, ETH_QOS_KCFG_BUF_SIZE, dpaa2_ni_dmamap_cb,
- &buf->store.paddr, BUS_DMA_NOWAIT);
+ error = bus_dmamap_load(buf->dmat, buf->dmap, buf->vaddr,
+ ETH_QOS_KCFG_BUF_SIZE, dpaa2_dmamap_oneseg_cb, &buf->paddr,
+ BUS_DMA_NOWAIT);
if (error) {
device_printf(dev, "%s: failed to map QoS key configuration "
"buffer into bus space\n", __func__);
@@ -2430,7 +2137,7 @@
tbl.default_tc = 0;
tbl.discard_on_miss = false;
tbl.keep_entries = false;
- tbl.kcfg_busaddr = buf->store.paddr;
+ tbl.kcfg_busaddr = buf->paddr;
error = DPAA2_CMD_NI_SET_QOS_TABLE(dev, child, &cmd, &tbl);
if (error) {
device_printf(dev, "%s: failed to set QoS table\n", __func__);
@@ -2761,22 +2468,22 @@
dpaa2_ni_transmit(if_t ifp, struct mbuf *m)
{
struct dpaa2_ni_softc *sc = if_getsoftc(ifp);
- struct dpaa2_ni_channel *chan;
- struct dpaa2_ni_tx_ring *tx;
+ struct dpaa2_channel *ch;
uint32_t fqid;
bool found = false;
- int chan_n = 0;
+ int chidx = 0, error;
- if (__predict_false(!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)))
+ if (__predict_false(!(if_getdrvflags(ifp) & IFF_DRV_RUNNING))) {
return (0);
+ }
if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) {
fqid = m->m_pkthdr.flowid;
for (int i = 0; i < sc->chan_n; i++) {
- chan = sc->channels[i];
- for (int j = 0; j < chan->rxq_n; j++) {
- if (fqid == chan->rx_queues[j].fqid) {
- chan_n = chan->flowid;
+ ch = sc->channels[i];
+ for (int j = 0; j < ch->rxq_n; j++) {
+ if (fqid == ch->rx_queues[j].fqid) {
+ chidx = ch->flowid;
found = true;
break;
}
@@ -2786,13 +2493,16 @@
}
}
}
- tx = DPAA2_TX_RING(sc, chan_n, 0);
- TX_LOCK(tx);
- dpaa2_ni_tx_locked(sc, tx, m);
- TX_UNLOCK(tx);
+ ch = sc->channels[chidx];
+ error = buf_ring_enqueue(ch->xmit_br, m);
+ if (__predict_false(error != 0)) {
+ m_freem(m);
+ } else {
+ taskqueue_enqueue(ch->cleanup_tq, &ch->cleanup_task);
+ }
- return (0);
+ return (error);
}
static void
@@ -3092,175 +2802,172 @@
}
/**
- * @brief Callback to obtain a physical address of the only DMA segment mapped.
+ * @brief Execute channel's Rx/Tx routines.
+ *
+ * NOTE: Should not be re-entrant for the same channel. It is achieved by
+ * enqueuing the cleanup routine on a single-threaded taskqueue.
*/
static void
-dpaa2_ni_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
+dpaa2_ni_cleanup_task(void *arg, int count)
{
- if (error == 0) {
- KASSERT(nseg == 1, ("too many segments: nseg=%d\n", nseg));
- *(bus_addr_t *) arg = segs[0].ds_addr;
- }
-}
+ struct dpaa2_channel *ch = (struct dpaa2_channel *)arg;
+ struct dpaa2_ni_softc *sc = device_get_softc(ch->ni_dev);
+ int error, rxc, txc;
-/**
- * @brief Release new buffers to the buffer pool if necessary.
- */
-static void
-dpaa2_ni_bp_task(void *arg, int count)
-{
- device_t bp_dev;
- struct dpaa2_ni_softc *sc = (struct dpaa2_ni_softc *) arg;
- struct dpaa2_bp_softc *bpsc;
- struct dpaa2_bp_conf bp_conf;
- const int buf_num = DPAA2_ATOMIC_READ(&sc->buf_num);
- int error;
+ for (int i = 0; i < DPAA2_CLEAN_BUDGET; i++) {
+ rxc = dpaa2_ni_rx_cleanup(ch);
+ txc = dpaa2_ni_tx_cleanup(ch);
- /* There's only one buffer pool for now. */
- bp_dev = (device_t) rman_get_start(sc->res[BP_RID(0)]);
- bpsc = device_get_softc(bp_dev);
+ if (__predict_false((if_getdrvflags(sc->ifp) &
+ IFF_DRV_RUNNING) == 0)) {
+ return;
+ }
- /* Get state of the buffer pool. */
- error = DPAA2_SWP_QUERY_BP(sc->channels[0]->io_dev, bpsc->attr.bpid,
- &bp_conf);
- if (error) {
- device_printf(sc->dev, "%s: failed to query buffer pool "
- "configuration: error=%d\n", __func__, error);
- return;
+ if ((txc != DPAA2_TX_BUDGET) && (rxc != DPAA2_RX_BUDGET)) {
+ break;
+ }
}
- /* Double allocated buffers number if free buffers < 25%. */
- if (bp_conf.free_bufn < (buf_num >> 2)) {
- (void)dpaa2_ni_seed_buf_pool(sc, buf_num);
- DPAA2_ATOMIC_XCHG(&sc->buf_free, bp_conf.free_bufn);
+ /* Re-arm channel to generate CDAN */
+ error = DPAA2_SWP_CONF_WQ_CHANNEL(ch->io_dev, &ch->ctx);
+ if (error != 0) {
+ panic("%s: failed to rearm channel: chan_id=%d, error=%d\n",
+ __func__, ch->id, error);
}
}
/**
* @brief Poll frames from a specific channel when CDAN is received.
- *
- * NOTE: To be called from the DPIO interrupt handler.
*/
-static void
-dpaa2_ni_poll(void *arg)
+static int
+dpaa2_ni_rx_cleanup(struct dpaa2_channel *ch)
{
- struct dpaa2_ni_channel *chan = (struct dpaa2_ni_channel *) arg;
- struct dpaa2_io_softc *iosc;
- struct dpaa2_swp *swp;
+ struct dpaa2_io_softc *iosc = device_get_softc(ch->io_dev);
+ struct dpaa2_swp *swp = iosc->swp;
struct dpaa2_ni_fq *fq;
+ struct dpaa2_buf *buf = &ch->store;
+ int budget = DPAA2_RX_BUDGET;
int error, consumed = 0;
- KASSERT(chan != NULL, ("%s: channel is NULL", __func__));
-
- iosc = device_get_softc(chan->io_dev);
- swp = iosc->swp;
-
do {
- error = dpaa2_swp_pull(swp, chan->id, &chan->store,
- ETH_STORE_FRAMES);
+ error = dpaa2_swp_pull(swp, ch->id, buf, DPAA2_ETH_STORE_FRAMES);
if (error) {
- device_printf(chan->ni_dev, "%s: failed to pull frames: "
- "chan_id=%d, error=%d\n", __func__, chan->id, error);
+ device_printf(ch->ni_dev, "%s: failed to pull frames: "
+ "chan_id=%d, error=%d\n", __func__, ch->id, error);
break;
}
-
- /*
- * TODO: Combine frames from the same Rx queue returned as
- * a result to the current VDQ command into a chain (linked
- * with m_nextpkt) to ammortize the FQ lock.
- */
- error = dpaa2_ni_consume_frames(chan, &fq, &consumed);
- if (error == ENOENT) {
+ error = dpaa2_ni_consume_frames(ch, &fq, &consumed);
+ if (error == ENOENT || error == EALREADY) {
break;
}
if (error == ETIMEDOUT) {
- device_printf(chan->ni_dev, "%s: timeout to consume "
- "frames: chan_id=%d\n", __func__, chan->id);
+ device_printf(ch->ni_dev, "%s: timeout to consume "
+ "frames: chan_id=%d\n", __func__, ch->id);
}
- } while (true);
+ } while (--budget);
- /* Re-arm channel to generate CDAN. */
- error = DPAA2_SWP_CONF_WQ_CHANNEL(chan->io_dev, &chan->ctx);
- if (error) {
- device_printf(chan->ni_dev, "%s: failed to rearm: chan_id=%d, "
- "error=%d\n", __func__, chan->id, error);
- }
+ return (DPAA2_RX_BUDGET - budget);
+}
+
+static int
+dpaa2_ni_tx_cleanup(struct dpaa2_channel *ch)
+{
+ struct dpaa2_ni_softc *sc = device_get_softc(ch->ni_dev);
+ struct dpaa2_ni_tx_ring *tx = &ch->txc_queue.tx_rings[0];
+ struct mbuf *m = NULL;
+ int budget = DPAA2_TX_BUDGET;
+
+ do {
+ mtx_assert(&ch->xmit_mtx, MA_NOTOWNED);
+ mtx_lock(&ch->xmit_mtx);
+ m = buf_ring_dequeue_sc(ch->xmit_br);
+ mtx_unlock(&ch->xmit_mtx);
+
+ if (__predict_false(m == NULL)) {
+ /* TODO: Do not give up easily */
+ break;
+ } else {
+ dpaa2_ni_tx(sc, ch, tx, m);
+ }
+ } while (--budget);
+
+ return (DPAA2_TX_BUDGET - budget);
}
-/**
- * @brief Transmit mbufs.
- */
static void
-dpaa2_ni_tx_locked(struct dpaa2_ni_softc *sc, struct dpaa2_ni_tx_ring *tx,
- struct mbuf *m)
+dpaa2_ni_tx(struct dpaa2_ni_softc *sc, struct dpaa2_channel *ch,
+ struct dpaa2_ni_tx_ring *tx, struct mbuf *m)
{
+ device_t dev = sc->dev;
struct dpaa2_ni_fq *fq = tx->fq;
- struct dpaa2_buf *buf;
+ struct dpaa2_buf *buf, *sgt;
struct dpaa2_fd fd;
- struct mbuf *m_d;
- bus_dma_segment_t txsegs[DPAA2_TX_SEGLIMIT];
- uint64_t idx;
- void *pidx;
- int error, rc, txnsegs;
-
- /* Obtain an index of a Tx buffer. */
- pidx = buf_ring_dequeue_sc(tx->idx_br);
- if (__predict_false(pidx == NULL)) {
- /* TODO: Do not give up easily. */
+ struct mbuf *md;
+ bus_dma_segment_t segs[DPAA2_TX_SEGLIMIT];
+ int rc, nsegs;
+ int error;
+
+ mtx_assert(&tx->lock, MA_NOTOWNED);
+ mtx_lock(&tx->lock);
+ buf = buf_ring_dequeue_sc(tx->br);
+ mtx_unlock(&tx->lock);
+ if (__predict_false(buf == NULL)) {
+ /* TODO: Do not give up easily */
m_freem(m);
return;
} else {
- idx = (uint64_t) pidx;
- buf = &tx->buf[idx];
- buf->tx.m = m;
- buf->tx.sgt_paddr = 0;
+ DPAA2_BUF_ASSERT_TXREADY(buf);
+ buf->m = m;
+ sgt = buf->sgt;
}
- /* Load mbuf to transmit. */
- error = bus_dmamap_load_mbuf_sg(buf->tx.dmat, buf->tx.dmap, m,
- txsegs, &txnsegs, BUS_DMA_NOWAIT);
+#if defined(INVARIANTS)
+ struct dpaa2_ni_tx_ring *btx = (struct dpaa2_ni_tx_ring *)buf->opt;
+ KASSERT(buf->opt == tx, ("%s: unexpected Tx ring", __func__));
+ KASSERT(btx->fq->chan == ch, ("%s: unexpected channel", __func__));
+#endif /* INVARIANTS */
+
+ error = bus_dmamap_load_mbuf_sg(buf->dmat, buf->dmap, m, segs, &nsegs,
+ BUS_DMA_NOWAIT);
if (__predict_false(error != 0)) {
/* Too many fragments, trying to defragment... */
- m_d = m_collapse(m, M_NOWAIT, DPAA2_TX_SEGLIMIT);
- if (m_d == NULL) {
- device_printf(sc->dev, "%s: mbuf "
- "defragmentation failed\n", __func__);
+ md = m_collapse(m, M_NOWAIT, DPAA2_TX_SEGLIMIT);
+ if (md == NULL) {
+ device_printf(dev, "%s: m_collapse() failed\n", __func__);
fq->chan->tx_dropped++;
goto err;
}
- buf->tx.m = m = m_d;
- error = bus_dmamap_load_mbuf_sg(buf->tx.dmat,
- buf->tx.dmap, m, txsegs, &txnsegs, BUS_DMA_NOWAIT);
+ buf->m = m = md;
+ error = bus_dmamap_load_mbuf_sg(buf->dmat, buf->dmap, m, segs,
+ &nsegs, BUS_DMA_NOWAIT);
if (__predict_false(error != 0)) {
- device_printf(sc->dev, "%s: failed to load "
- "mbuf: error=%d\n", __func__, error);
+ device_printf(dev, "%s: bus_dmamap_load_mbuf_sg() "
+ "failed: error=%d\n", __func__, error);
fq->chan->tx_dropped++;
goto err;
}
}
- /* Build frame descriptor. */
- error = dpaa2_ni_build_fd(sc, tx, buf, txsegs, txnsegs, &fd);
+ error = dpaa2_ni_build_fd(sc, tx, buf, segs, nsegs, &fd);
if (__predict_false(error != 0)) {
- device_printf(sc->dev, "%s: failed to build frame "
- "descriptor: error=%d\n", __func__, error);
+ device_printf(dev, "%s: failed to build frame descriptor: "
+ "error=%d\n", __func__, error);
fq->chan->tx_dropped++;
goto err_unload;
}
- /* TODO: Enqueue several frames in a single command. */
+ /* TODO: Enqueue several frames in a single command */
for (int i = 0; i < DPAA2_NI_ENQUEUE_RETRIES; i++) {
- /* TODO: Return error codes instead of # of frames. */
- rc = DPAA2_SWP_ENQ_MULTIPLE_FQ(fq->chan->io_dev, tx->fqid,
- &fd, 1);
+ /* TODO: Return error codes instead of # of frames */
+ rc = DPAA2_SWP_ENQ_MULTIPLE_FQ(fq->chan->io_dev, tx->fqid, &fd, 1);
if (rc == 1) {
break;
}
}
- bus_dmamap_sync(buf->tx.dmat, buf->tx.dmap, BUS_DMASYNC_PREWRITE);
- bus_dmamap_sync(buf->tx.sgt_dmat, buf->tx.sgt_dmap, BUS_DMASYNC_PREWRITE);
+ bus_dmamap_sync(buf->dmat, buf->dmap, BUS_DMASYNC_PREWRITE);
+ bus_dmamap_sync(sgt->dmat, sgt->dmap, BUS_DMASYNC_PREWRITE);
if (rc != 1) {
fq->chan->tx_dropped++;
@@ -3271,58 +2978,98 @@
return;
err_unload:
- bus_dmamap_unload(buf->tx.dmat, buf->tx.dmap);
- if (buf->tx.sgt_paddr != 0) {
- bus_dmamap_unload(buf->tx.sgt_dmat, buf->tx.sgt_dmap);
+ bus_dmamap_unload(buf->dmat, buf->dmap);
+ if (sgt->paddr != 0) {
+ bus_dmamap_unload(sgt->dmat, sgt->dmap);
}
err:
- m_freem(buf->tx.m);
- buf_ring_enqueue(tx->idx_br, pidx);
+ m_freem(buf->m);
+ buf_ring_enqueue(tx->br, buf);
}
static int
-dpaa2_ni_consume_frames(struct dpaa2_ni_channel *chan, struct dpaa2_ni_fq **src,
+dpaa2_ni_consume_frames(struct dpaa2_channel *chan, struct dpaa2_ni_fq **src,
uint32_t *consumed)
{
struct dpaa2_ni_fq *fq = NULL;
struct dpaa2_dq *dq;
struct dpaa2_fd *fd;
+ struct dpaa2_ni_rx_ctx ctx = {
+ .head = NULL,
+ .tail = NULL,
+ .cnt = 0,
+ .last = false
+ };
int rc, frames = 0;
do {
- rc = dpaa2_ni_chan_storage_next(chan, &dq);
+ rc = dpaa2_chan_next_frame(chan, &dq);
if (rc == EINPROGRESS) {
if (dq != NULL && !IS_NULL_RESPONSE(dq->fdr.desc.stat)) {
fd = &dq->fdr.fd;
fq = (struct dpaa2_ni_fq *) dq->fdr.desc.fqd_ctx;
- fq->consume(chan, fq, fd);
+
+ switch (fq->type) {
+ case DPAA2_NI_QUEUE_RX:
+ (void)dpaa2_ni_rx(chan, fq, fd, &ctx);
+ break;
+ case DPAA2_NI_QUEUE_RX_ERR:
+ (void)dpaa2_ni_rx_err(chan, fq, fd);
+ break;
+ case DPAA2_NI_QUEUE_TX_CONF:
+ (void)dpaa2_ni_tx_conf(chan, fq, fd);
+ break;
+ default:
+ panic("%s: unknown queue type (1)",
+ __func__);
+ }
frames++;
}
} else if (rc == EALREADY || rc == ENOENT) {
if (dq != NULL && !IS_NULL_RESPONSE(dq->fdr.desc.stat)) {
fd = &dq->fdr.fd;
fq = (struct dpaa2_ni_fq *) dq->fdr.desc.fqd_ctx;
- fq->consume(chan, fq, fd);
+
+ switch (fq->type) {
+ case DPAA2_NI_QUEUE_RX:
+ /*
+ * Last VDQ response (mbuf) in a chain
+ * obtained from the Rx queue.
+ */
+ ctx.last = true;
+ (void)dpaa2_ni_rx(chan, fq, fd, &ctx);
+ break;
+ case DPAA2_NI_QUEUE_RX_ERR:
+ (void)dpaa2_ni_rx_err(chan, fq, fd);
+ break;
+ case DPAA2_NI_QUEUE_TX_CONF:
+ (void)dpaa2_ni_tx_conf(chan, fq, fd);
+ break;
+ default:
+ panic("%s: unknown queue type (2)",
+ __func__);
+ }
frames++;
}
break;
} else {
- KASSERT(1 == 0, ("%s: should not reach here", __func__));
+ panic("%s: should not reach here: rc=%d", __func__, rc);
}
} while (true);
- KASSERT(chan->store_idx < chan->store_sz,
- ("channel store idx >= size: store_idx=%d, store_sz=%d",
- chan->store_idx, chan->store_sz));
+ KASSERT(chan->store_idx < chan->store_sz, ("%s: store_idx(%d) >= "
+ "store_sz(%d)", __func__, chan->store_idx, chan->store_sz));
/*
- * A dequeue operation pulls frames from a single queue into the store.
+ * VDQ operation pulls frames from a single queue into the store.
* Return the frame queue and a number of consumed frames as an output.
*/
- if (src != NULL)
+ if (src != NULL) {
*src = fq;
- if (consumed != NULL)
+ }
+ if (consumed != NULL) {
*consumed = frames;
+ }
return (rc);
}
@@ -3331,32 +3078,33 @@
* @brief Receive frames.
*/
static int
-dpaa2_ni_rx(struct dpaa2_ni_channel *chan, struct dpaa2_ni_fq *fq,
- struct dpaa2_fd *fd)
+dpaa2_ni_rx(struct dpaa2_channel *ch, struct dpaa2_ni_fq *fq, struct dpaa2_fd *fd,
+ struct dpaa2_ni_rx_ctx *ctx)
{
- struct dpaa2_ni_softc *sc = device_get_softc(chan->ni_dev);
+ bus_addr_t paddr = (bus_addr_t)fd->addr;
+ struct dpaa2_fa *fa = (struct dpaa2_fa *)PHYS_TO_DMAP(paddr);
+ struct dpaa2_buf *buf = fa->buf;
+ struct dpaa2_channel *bch = (struct dpaa2_channel *)buf->opt;
+ struct dpaa2_ni_softc *sc = device_get_softc(bch->ni_dev);
struct dpaa2_bp_softc *bpsc;
- struct dpaa2_buf *buf;
- struct dpaa2_fa *fa;
- if_t ifp = sc->ifp;
struct mbuf *m;
- device_t bp_dev;
- bus_addr_t paddr = (bus_addr_t) fd->addr;
+ device_t bpdev;
bus_addr_t released[DPAA2_SWP_BUFS_PER_CMD];
void *buf_data;
int buf_len, error, released_n = 0;
- fa = (struct dpaa2_fa *) PHYS_TO_DMAP(paddr);
- buf = fa->buf;
-
KASSERT(fa->magic == DPAA2_MAGIC, ("%s: wrong magic", __func__));
- KASSERT(buf->type == DPAA2_BUF_RX, ("%s: not Rx buffer", __func__));
- if (__predict_false(paddr != buf->rx.paddr)) {
+ /*
+ * NOTE: Current channel might not be the same as the "buffer" channel
+ * and it's fine. It must not be NULL though.
+ */
+ KASSERT(bch != NULL, ("%s: buffer channel is NULL", __func__));
+
+ if (__predict_false(paddr != buf->paddr)) {
panic("%s: unexpected physical address: fd(%#jx) != buf(%#jx)",
- __func__, paddr, buf->rx.paddr);
+ __func__, paddr, buf->paddr);
}
- /* Update statistics. */
switch (dpaa2_ni_fd_err(fd)) {
case 1: /* Enqueue rejected by QMan */
sc->rx_enq_rej_frames++;
@@ -3378,70 +3126,88 @@
break;
}
- m = buf->rx.m;
- buf->rx.m = NULL;
- bus_dmamap_sync(buf->rx.dmat, buf->rx.dmap, BUS_DMASYNC_POSTREAD);
- bus_dmamap_unload(buf->rx.dmat, buf->rx.dmap);
+ mtx_assert(&bch->dma_mtx, MA_NOTOWNED);
+ mtx_lock(&bch->dma_mtx);
+ bus_dmamap_sync(buf->dmat, buf->dmap, BUS_DMASYNC_POSTREAD);
+ bus_dmamap_unload(buf->dmat, buf->dmap);
+ m = buf->m;
buf_len = dpaa2_ni_fd_data_len(fd);
- buf_data = (uint8_t *)buf->rx.vaddr + dpaa2_ni_fd_offset(fd);
+ buf_data = (uint8_t *)buf->vaddr + dpaa2_ni_fd_offset(fd);
+ /* Prepare buffer to be re-cycled */
+ buf->m = NULL;
+ buf->paddr = 0;
+ buf->vaddr = NULL;
+ buf->seg.ds_addr = 0;
+ buf->seg.ds_len = 0;
+ buf->nseg = 0;
- /* Prefetch mbuf data. */
- __builtin_prefetch(buf_data);
+ mtx_unlock(&bch->dma_mtx);
- /* Write value to mbuf (avoid reading). */
m->m_flags |= M_PKTHDR;
m->m_data = buf_data;
m->m_len = buf_len;
m->m_pkthdr.len = buf_len;
- m->m_pkthdr.rcvif = ifp;
+ m->m_pkthdr.rcvif = sc->ifp;
m->m_pkthdr.flowid = fq->fqid;
M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
- if_input(ifp, m);
+ if (ctx->head == NULL) {
+ KASSERT(ctx->tail == NULL, ("%s: tail already given?", __func__));
+ ctx->head = m;
+ ctx->tail = m;
+ } else {
+ KASSERT(ctx->head != NULL, ("%s: head is NULL", __func__));
+ ctx->tail->m_nextpkt = m;
+ ctx->tail = m;
+ }
+ ctx->cnt++;
+
+ if (ctx->last) {
+ ctx->tail->m_nextpkt = NULL;
+ if_input(sc->ifp, ctx->head);
+ }
+
+ /* Keep the buffer to be recycled */
+ ch->recycled[ch->recycled_n++] = buf;
- /* Keep the buffer to be recycled. */
- chan->recycled[chan->recycled_n++] = buf;
- KASSERT(chan->recycled_n <= DPAA2_SWP_BUFS_PER_CMD,
- ("%s: too many buffers to recycle", __func__));
+ /* Re-seed and release recycled buffers back to the pool */
+ if (ch->recycled_n == DPAA2_SWP_BUFS_PER_CMD) {
+ /* Release new buffers to the pool if needed */
+ taskqueue_enqueue(sc->bp_taskq, &ch->bp_task);
- /* Re-seed and release recycled buffers back to the pool. */
- if (chan->recycled_n == DPAA2_SWP_BUFS_PER_CMD) {
- /* Release new buffers to the pool if needed. */
- taskqueue_enqueue(sc->bp_taskq, &sc->bp_task);
+ for (int i = 0; i < ch->recycled_n; i++) {
+ buf = ch->recycled[i];
+ bch = (struct dpaa2_channel *)buf->opt;
- for (int i = 0; i < chan->recycled_n; i++) {
- buf = chan->recycled[i];
+ mtx_assert(&bch->dma_mtx, MA_NOTOWNED);
+ mtx_lock(&bch->dma_mtx);
+ error = dpaa2_buf_seed_rxb(sc->dev, buf,
+ DPAA2_RX_BUF_SIZE, &bch->dma_mtx);
+ mtx_unlock(&bch->dma_mtx);
- /* Seed recycled buffer. */
- error = dpaa2_ni_seed_rxbuf(sc, buf);
- KASSERT(error == 0, ("%s: failed to seed recycled "
- "buffer: error=%d", __func__, error));
if (__predict_false(error != 0)) {
- device_printf(sc->dev, "%s: failed to seed "
- "recycled buffer: error=%d\n", __func__,
- error);
- continue;
+ /* TODO: What else to do with the buffer? */
+ panic("%s: failed to recycle buffer: error=%d",
+ __func__, error);
}
- /* Prepare buffer to be released in a single command. */
- released[released_n++] = buf->rx.paddr;
+ /* Prepare buffer to be released in a single command */
+ released[released_n++] = buf->paddr;
}
- /* There's only one buffer pool for now. */
- bp_dev = (device_t) rman_get_start(sc->res[BP_RID(0)]);
- bpsc = device_get_softc(bp_dev);
+ /* There's only one buffer pool for now */
+ bpdev = (device_t)rman_get_start(sc->res[DPAA2_NI_BP_RID(0)]);
+ bpsc = device_get_softc(bpdev);
- error = DPAA2_SWP_RELEASE_BUFS(chan->io_dev, bpsc->attr.bpid,
+ error = DPAA2_SWP_RELEASE_BUFS(ch->io_dev, bpsc->attr.bpid,
released, released_n);
if (__predict_false(error != 0)) {
device_printf(sc->dev, "%s: failed to release buffers "
"to the pool: error=%d\n", __func__, error);
return (error);
}
-
- /* Be ready to recycle the next portion of the buffers. */
- chan->recycled_n = 0;
+ ch->recycled_n = 0;
}
return (0);
@@ -3451,33 +3217,36 @@
* @brief Receive Rx error frames.
*/
static int
-dpaa2_ni_rx_err(struct dpaa2_ni_channel *chan, struct dpaa2_ni_fq *fq,
+dpaa2_ni_rx_err(struct dpaa2_channel *ch, struct dpaa2_ni_fq *fq,
struct dpaa2_fd *fd)
{
- device_t bp_dev;
- struct dpaa2_ni_softc *sc = device_get_softc(chan->ni_dev);
+ bus_addr_t paddr = (bus_addr_t)fd->addr;
+ struct dpaa2_fa *fa = (struct dpaa2_fa *)PHYS_TO_DMAP(paddr);
+ struct dpaa2_buf *buf = fa->buf;
+ struct dpaa2_channel *bch = (struct dpaa2_channel *)buf->opt;
+ struct dpaa2_ni_softc *sc = device_get_softc(bch->ni_dev);
+ device_t bpdev;
struct dpaa2_bp_softc *bpsc;
- struct dpaa2_buf *buf;
- struct dpaa2_fa *fa;
- bus_addr_t paddr = (bus_addr_t) fd->addr;
int error;
- fa = (struct dpaa2_fa *) PHYS_TO_DMAP(paddr);
- buf = fa->buf;
-
KASSERT(fa->magic == DPAA2_MAGIC, ("%s: wrong magic", __func__));
- KASSERT(buf->type == DPAA2_BUF_RX, ("%s: not Rx buffer", __func__));
- if (__predict_false(paddr != buf->rx.paddr)) {
+ /*
+ * NOTE: Current channel might not be the same as the "buffer" channel
+ * and it's fine. It must not be NULL though.
+ */
+ KASSERT(bch != NULL, ("%s: buffer channel is NULL", __func__));
+
+ if (__predict_false(paddr != buf->paddr)) {
panic("%s: unexpected physical address: fd(%#jx) != buf(%#jx)",
- __func__, paddr, buf->rx.paddr);
+ __func__, paddr, buf->paddr);
}
- /* There's only one buffer pool for now. */
- bp_dev = (device_t) rman_get_start(sc->res[BP_RID(0)]);
- bpsc = device_get_softc(bp_dev);
+ /* There's only one buffer pool for now */
+ bpdev = (device_t)rman_get_start(sc->res[DPAA2_NI_BP_RID(0)]);
+ bpsc = device_get_softc(bpdev);
- /* Release buffer to QBMan buffer pool. */
- error = DPAA2_SWP_RELEASE_BUFS(chan->io_dev, bpsc->attr.bpid, &paddr, 1);
+ /* Release buffer to QBMan buffer pool */
+ error = DPAA2_SWP_RELEASE_BUFS(ch->io_dev, bpsc->attr.bpid, &paddr, 1);
if (error != 0) {
device_printf(sc->dev, "%s: failed to release frame buffer to "
"the pool: error=%d\n", __func__, error);
@@ -3491,33 +3260,47 @@
* @brief Receive Tx confirmation frames.
*/
static int
-dpaa2_ni_tx_conf(struct dpaa2_ni_channel *chan, struct dpaa2_ni_fq *fq,
+dpaa2_ni_tx_conf(struct dpaa2_channel *ch, struct dpaa2_ni_fq *fq,
struct dpaa2_fd *fd)
{
- struct dpaa2_ni_tx_ring *tx;
- struct dpaa2_buf *buf;
- struct dpaa2_fa *fa;
- bus_addr_t paddr = (bus_addr_t) fd->addr;
-
- fa = (struct dpaa2_fa *) PHYS_TO_DMAP(paddr);
- buf = fa->buf;
- tx = fa->tx;
+ bus_addr_t paddr = (bus_addr_t)fd->addr;
+ struct dpaa2_fa *fa = (struct dpaa2_fa *)PHYS_TO_DMAP(paddr);
+ struct dpaa2_buf *buf = fa->buf;
+ struct dpaa2_buf *sgt = buf->sgt;
+ struct dpaa2_ni_tx_ring *tx = (struct dpaa2_ni_tx_ring *)buf->opt;
+ struct dpaa2_channel *bch = tx->fq->chan;
KASSERT(fa->magic == DPAA2_MAGIC, ("%s: wrong magic", __func__));
- KASSERT(buf->type == DPAA2_BUF_TX, ("%s: not Tx buffer", __func__));
- if (paddr != buf->tx.paddr) {
+ KASSERT(tx != NULL, ("%s: Tx ring is NULL", __func__));
+ KASSERT(sgt != NULL, ("%s: S/G table is NULL", __func__));
+ /*
+ * NOTE: Current channel might not be the same as the "buffer" channel
+ * and it's fine. It must not be NULL though.
+ */
+ KASSERT(bch != NULL, ("%s: buffer channel is NULL", __func__));
+
+ if (paddr != buf->paddr) {
panic("%s: unexpected physical address: fd(%#jx) != buf(%#jx)",
- __func__, paddr, buf->tx.paddr);
+ __func__, paddr, buf->paddr);
}
- bus_dmamap_sync(buf->tx.dmat, buf->tx.dmap, BUS_DMASYNC_POSTWRITE);
- bus_dmamap_sync(buf->tx.sgt_dmat, buf->tx.sgt_dmap, BUS_DMASYNC_POSTWRITE);
- bus_dmamap_unload(buf->tx.dmat, buf->tx.dmap);
- bus_dmamap_unload(buf->tx.sgt_dmat, buf->tx.sgt_dmap);
- m_freem(buf->tx.m);
+ mtx_assert(&bch->dma_mtx, MA_NOTOWNED);
+ mtx_lock(&bch->dma_mtx);
- /* Return Tx buffer index back to the ring. */
- buf_ring_enqueue(tx->idx_br, (void *) buf->tx.idx);
+ bus_dmamap_sync(buf->dmat, buf->dmap, BUS_DMASYNC_POSTWRITE);
+ bus_dmamap_sync(sgt->dmat, sgt->dmap, BUS_DMASYNC_POSTWRITE);
+ bus_dmamap_unload(buf->dmat, buf->dmap);
+ bus_dmamap_unload(sgt->dmat, sgt->dmap);
+ m_freem(buf->m);
+ buf->m = NULL;
+ buf->paddr = 0;
+ buf->vaddr = NULL;
+ sgt->paddr = 0;
+
+ mtx_unlock(&bch->dma_mtx);
+
+ /* Return Tx buffer back to the ring */
+ buf_ring_enqueue(tx->br, buf);
return (0);
}
@@ -3529,240 +3312,10 @@
dpaa2_ni_cmp_api_version(struct dpaa2_ni_softc *sc, uint16_t major,
uint16_t minor)
{
- if (sc->api_major == major)
+ if (sc->api_major == major) {
return sc->api_minor - minor;
- return sc->api_major - major;
-}
-
-/**
- * @brief Allocate Rx buffers visible to QBMan and release them to the pool.
- */
-static int
-dpaa2_ni_seed_buf_pool(struct dpaa2_ni_softc *sc, uint32_t seedn)
-{
- device_t bp_dev;
- struct dpaa2_bp_softc *bpsc;
- struct dpaa2_buf *buf;
- bus_addr_t paddr[DPAA2_SWP_BUFS_PER_CMD];
- const int allocated = DPAA2_ATOMIC_READ(&sc->buf_num);
- int i, error, bufn = 0;
-
- KASSERT(sc->bp_dmat != NULL, ("%s: DMA tag for buffer pool not "
- "created?", __func__));
-
- /* There's only one buffer pool for now. */
- bp_dev = (device_t) rman_get_start(sc->res[BP_RID(0)]);
- bpsc = device_get_softc(bp_dev);
-
- /* Limit # of buffers released to the pool. */
- if (allocated + seedn > DPAA2_NI_BUFS_MAX)
- seedn = DPAA2_NI_BUFS_MAX - allocated;
-
- /* Release "seedn" buffers to the pool. */
- for (i = allocated; i < (allocated + seedn); i++) {
- /* Enough buffers were allocated for a single command. */
- if (bufn == DPAA2_SWP_BUFS_PER_CMD) {
- error = DPAA2_SWP_RELEASE_BUFS(sc->channels[0]->io_dev,
- bpsc->attr.bpid, paddr, bufn);
- if (error) {
- device_printf(sc->dev, "%s: failed to release "
- "buffers to the pool (1)\n", __func__);
- return (error);
- }
- DPAA2_ATOMIC_ADD(&sc->buf_num, bufn);
- bufn = 0;
- }
-
- buf = &sc->buf[i];
- buf->type = DPAA2_BUF_RX;
- buf->rx.m = NULL;
- buf->rx.dmap = NULL;
- buf->rx.paddr = 0;
- buf->rx.vaddr = NULL;
- error = dpaa2_ni_seed_rxbuf(sc, buf);
- if (error != 0) {
- break;
- }
- paddr[bufn] = buf->rx.paddr;
- bufn++;
- }
-
- /* Release if there are buffers left. */
- if (bufn > 0) {
- error = DPAA2_SWP_RELEASE_BUFS(sc->channels[0]->io_dev,
- bpsc->attr.bpid, paddr, bufn);
- if (error) {
- device_printf(sc->dev, "%s: failed to release "
- "buffers to the pool (2)\n", __func__);
- return (error);
- }
- DPAA2_ATOMIC_ADD(&sc->buf_num, bufn);
- }
-
- return (0);
-}
-
-/**
- * @brief Prepare Rx buffer to be released to the buffer pool.
- */
-static int
-dpaa2_ni_seed_rxbuf(struct dpaa2_ni_softc *sc, struct dpaa2_buf *buf)
-{
- struct mbuf *m;
- struct dpaa2_fa *fa;
- bus_dmamap_t dmap;
- bus_dma_segment_t segs;
- int error, nsegs;
-
- KASSERT(sc->bp_dmat != NULL, ("%s: Buffer pool DMA tag is not "
- "allocated?", __func__));
- KASSERT(buf->type == DPAA2_BUF_RX, ("%s: not Rx buffer", __func__));
-
- /* Keep DMA tag for this buffer. */
- if (__predict_false(buf->rx.dmat == NULL))
- buf->rx.dmat = sc->bp_dmat;
-
- /* Create a DMA map for the giving buffer if it doesn't exist yet. */
- if (__predict_false(buf->rx.dmap == NULL)) {
- error = bus_dmamap_create(buf->rx.dmat, 0, &dmap);
- if (error) {
- device_printf(sc->dev, "%s: failed to create DMA map "
- "for buffer: error=%d\n", __func__, error);
- return (error);
- }
- buf->rx.dmap = dmap;
- }
-
- /* Allocate mbuf if needed. */
- if (__predict_false(buf->rx.m == NULL)) {
- m = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, BUF_SIZE);
- if (__predict_false(m == NULL)) {
- device_printf(sc->dev, "%s: failed to allocate mbuf for "
- "buffer\n", __func__);
- return (ENOMEM);
- }
- m->m_len = m->m_ext.ext_size;
- m->m_pkthdr.len = m->m_ext.ext_size;
- buf->rx.m = m;
- } else
- m = buf->rx.m;
-
- error = bus_dmamap_load_mbuf_sg(buf->rx.dmat, buf->rx.dmap,
- m, &segs, &nsegs, BUS_DMA_NOWAIT);
- KASSERT(nsegs == 1, ("one segment expected: nsegs=%d", nsegs));
- KASSERT(error == 0, ("failed to map mbuf: error=%d", error));
- if (__predict_false(error != 0 || nsegs != 1)) {
- device_printf(sc->dev, "%s: failed to map mbuf: error=%d, "
- "nsegs=%d\n", __func__, error, nsegs);
- bus_dmamap_unload(buf->rx.dmat, buf->rx.dmap);
- m_freem(m);
- return (error);
- }
- buf->rx.paddr = segs.ds_addr;
- buf->rx.vaddr = m->m_data;
-
- /* Populate frame annotation for future use. */
- fa = (struct dpaa2_fa *) m->m_data;
- fa->magic = DPAA2_MAGIC;
- fa->buf = buf;
-
- bus_dmamap_sync(buf->rx.dmat, buf->rx.dmap, BUS_DMASYNC_PREREAD);
-
- return (0);
-}
-
-/**
- * @brief Prepare Tx buffer to be added to the Tx ring.
- */
-static int
-dpaa2_ni_seed_txbuf(struct dpaa2_ni_softc *sc, struct dpaa2_buf *buf)
-{
- bus_dmamap_t dmap;
- int error;
-
- KASSERT(sc->tx_dmat != NULL, ("%s: Tx DMA tag is not allocated?",
- __func__));
- KASSERT(sc->sgt_dmat != NULL, ("%s: S/G DMA tag not allocated?",
- __func__));
- KASSERT(buf->type == DPAA2_BUF_TX, ("%s: not Tx buffer", __func__));
-
- /* Keep DMA tags for this buffer. */
- if (__predict_true(buf->tx.dmat == NULL))
- buf->tx.dmat = sc->tx_dmat;
- if (__predict_true(buf->tx.sgt_dmat == NULL))
- buf->tx.sgt_dmat = sc->sgt_dmat;
-
- /* Create a DMA map for the giving buffer if it doesn't exist yet. */
- if (__predict_true(buf->tx.dmap == NULL)) {
- error = bus_dmamap_create(buf->tx.dmat, 0, &dmap);
- if (error != 0) {
- device_printf(sc->dev, "%s: failed to create "
- "Tx DMA map: error=%d\n", __func__, error);
- return (error);
- }
- buf->tx.dmap = dmap;
- }
-
- /* Allocate a buffer to store scatter/gather table. */
- if (__predict_true(buf->tx.sgt_vaddr == NULL)) {
- error = bus_dmamem_alloc(buf->tx.sgt_dmat,
- &buf->tx.sgt_vaddr, BUS_DMA_ZERO | BUS_DMA_COHERENT,
- &buf->tx.sgt_dmap);
- if (error != 0) {
- device_printf(sc->dev, "%s: failed to allocate "
- "S/G table: error=%d\n", __func__, error);
- return (error);
- }
- }
-
- return (0);
-}
-
-/**
- * @brief Allocate channel storage visible to QBMan.
- */
-static int
-dpaa2_ni_seed_chan_storage(struct dpaa2_ni_softc *sc,
- struct dpaa2_ni_channel *chan)
-{
- struct dpaa2_buf *buf = &chan->store;
- int error;
-
- KASSERT(sc->st_dmat != NULL, ("%s: channel storage DMA tag is not "
- "allocated?", __func__));
- KASSERT(buf->type == DPAA2_BUF_STORE, ("%s: not channel storage buffer",
- __func__));
-
- /* Keep DMA tag for this buffer. */
- if (__predict_false(buf->store.dmat == NULL)) {
- buf->store.dmat = sc->st_dmat;
- }
-
- if (__predict_false(buf->store.vaddr == NULL)) {
- error = bus_dmamem_alloc(buf->store.dmat, &buf->store.vaddr,
- BUS_DMA_ZERO | BUS_DMA_COHERENT, &buf->store.dmap);
- if (error) {
- device_printf(sc->dev, "%s: failed to allocate channel "
- "storage\n", __func__);
- return (error);
- }
- }
-
- if (__predict_false(buf->store.paddr == 0)) {
- error = bus_dmamap_load(buf->store.dmat, buf->store.dmap,
- buf->store.vaddr, ETH_STORE_SIZE, dpaa2_ni_dmamap_cb,
- &buf->store.paddr, BUS_DMA_NOWAIT);
- if (error) {
- device_printf(sc->dev, "%s: failed to map channel "
- "storage\n", __func__);
- return (error);
- }
}
-
- chan->store_sz = ETH_STORE_FRAMES;
- chan->store_idx = 0;
-
- return (0);
+ return sc->api_major - major;
}
/**
@@ -3770,60 +3323,55 @@
*/
static int
dpaa2_ni_build_fd(struct dpaa2_ni_softc *sc, struct dpaa2_ni_tx_ring *tx,
- struct dpaa2_buf *buf, bus_dma_segment_t *txsegs, int txnsegs,
- struct dpaa2_fd *fd)
+ struct dpaa2_buf *buf, bus_dma_segment_t *segs, int nsegs, struct dpaa2_fd *fd)
{
- struct dpaa2_sg_entry *sgt;
+ struct dpaa2_buf *sgt = buf->sgt;
+ struct dpaa2_sg_entry *sge;
struct dpaa2_fa *fa;
int i, error;
- KASSERT(txnsegs <= DPAA2_TX_SEGLIMIT, ("%s: too many segments, "
- "txnsegs (%d) > %d", __func__, txnsegs, DPAA2_TX_SEGLIMIT));
- KASSERT(buf->type == DPAA2_BUF_TX, ("%s: not Tx buffer", __func__));
- KASSERT(buf->tx.sgt_vaddr != NULL, ("%s: S/G table not allocated?",
- __func__));
+ KASSERT(nsegs <= DPAA2_TX_SEGLIMIT, ("%s: too many segments", __func__));
+ KASSERT(buf->opt != NULL, ("%s: no Tx ring?", __func__));
+ KASSERT(sgt != NULL, ("%s: no S/G table?", __func__));
+ KASSERT(sgt->vaddr != NULL, ("%s: no S/G vaddr?", __func__));
- /* Reset frame descriptor fields. */
memset(fd, 0, sizeof(*fd));
- if (__predict_true(txnsegs <= DPAA2_TX_SEGLIMIT)) {
- /* Populate S/G table. */
- sgt = (struct dpaa2_sg_entry *) buf->tx.sgt_vaddr +
- sc->tx_data_off;
- for (i = 0; i < txnsegs; i++) {
- sgt[i].addr = (uint64_t) txsegs[i].ds_addr;
- sgt[i].len = (uint32_t) txsegs[i].ds_len;
- sgt[i].offset_fmt = 0u;
+ /* Populate and map S/G table */
+ if (__predict_true(nsegs <= DPAA2_TX_SEGLIMIT)) {
+ sge = (struct dpaa2_sg_entry *)sgt->vaddr + sc->tx_data_off;
+ for (i = 0; i < nsegs; i++) {
+ sge[i].addr = (uint64_t)segs[i].ds_addr;
+ sge[i].len = (uint32_t)segs[i].ds_len;
+ sge[i].offset_fmt = 0u;
}
- sgt[i-1].offset_fmt |= 0x8000u; /* set final entry flag */
+ sge[i-1].offset_fmt |= 0x8000u; /* set final entry flag */
- KASSERT(buf->tx.sgt_paddr == 0, ("%s: sgt_paddr(%#jx) != 0",
- __func__, buf->tx.sgt_paddr));
+ KASSERT(sgt->paddr == 0, ("%s: paddr(%#jx) != 0", __func__,
+ sgt->paddr));
- /* Load S/G table. */
- error = bus_dmamap_load(buf->tx.sgt_dmat, buf->tx.sgt_dmap,
- buf->tx.sgt_vaddr, DPAA2_TX_SGT_SZ, dpaa2_ni_dmamap_cb,
- &buf->tx.sgt_paddr, BUS_DMA_NOWAIT);
+ error = bus_dmamap_load(sgt->dmat, sgt->dmap, sgt->vaddr,
+ DPAA2_TX_SGT_SZ, dpaa2_dmamap_oneseg_cb, &sgt->paddr,
+ BUS_DMA_NOWAIT);
if (__predict_false(error != 0)) {
- device_printf(sc->dev, "%s: failed to map S/G table: "
+ device_printf(sc->dev, "%s: bus_dmamap_load() failed: "
"error=%d\n", __func__, error);
return (error);
}
- buf->tx.paddr = buf->tx.sgt_paddr;
- buf->tx.vaddr = buf->tx.sgt_vaddr;
+ buf->paddr = sgt->paddr;
+ buf->vaddr = sgt->vaddr;
sc->tx_sg_frames++; /* for sysctl(9) */
} else {
return (EINVAL);
}
- fa = (struct dpaa2_fa *) buf->tx.sgt_vaddr;
+ fa = (struct dpaa2_fa *)sgt->vaddr;
fa->magic = DPAA2_MAGIC;
fa->buf = buf;
- fa->tx = tx;
- fd->addr = buf->tx.paddr;
- fd->data_length = (uint32_t) buf->tx.m->m_pkthdr.len;
+ fd->addr = buf->paddr;
+ fd->data_length = (uint32_t)buf->m->m_pkthdr.len;
fd->bpid_ivp_bmt = 0;
fd->offset_fmt_sl = 0x2000u | sc->tx_data_off;
fd->ctrl = 0x00800000u;
@@ -3840,9 +3388,9 @@
static uint32_t
dpaa2_ni_fd_data_len(struct dpaa2_fd *fd)
{
- if (dpaa2_ni_fd_short_len(fd))
+ if (dpaa2_ni_fd_short_len(fd)) {
return (fd->data_length & DPAA2_NI_FD_LEN_MASK);
-
+ }
return (fd->data_length);
}
@@ -3969,10 +3517,8 @@
uint16_t rc_token, ni_token;
int i, error = 0;
- KASSERT(buf->type == DPAA2_BUF_STORE, ("%s: not storage buffer",
- __func__));
- if (__predict_true(buf->store.dmat == NULL)) {
- buf->store.dmat = sc->rxd_dmat;
+ if (__predict_true(buf->dmat == NULL)) {
+ buf->dmat = sc->rxd_dmat;
}
memset(&cls_cfg, 0, sizeof(cls_cfg));
@@ -3998,15 +3544,15 @@
cls_cfg.num_extracts++;
}
- error = bus_dmamem_alloc(buf->store.dmat, &buf->store.vaddr,
- BUS_DMA_ZERO | BUS_DMA_COHERENT, &buf->store.dmap);
+ error = bus_dmamem_alloc(buf->dmat, (void **)&buf->vaddr,
+ BUS_DMA_ZERO | BUS_DMA_COHERENT, &buf->dmap);
if (error != 0) {
device_printf(dev, "%s: failed to allocate a buffer for Rx "
"traffic distribution key configuration\n", __func__);
return (error);
}
- error = dpaa2_ni_prepare_key_cfg(&cls_cfg, (uint8_t *) buf->store.vaddr);
+ error = dpaa2_ni_prepare_key_cfg(&cls_cfg, (uint8_t *)buf->vaddr);
if (error != 0) {
device_printf(dev, "%s: failed to prepare key configuration: "
"error=%d\n", __func__, error);
@@ -4014,9 +3560,9 @@
}
/* Prepare for setting the Rx dist. */
- error = bus_dmamap_load(buf->store.dmat, buf->store.dmap,
- buf->store.vaddr, DPAA2_CLASSIFIER_DMA_SIZE, dpaa2_ni_dmamap_cb,
- &buf->store.paddr, BUS_DMA_NOWAIT);
+ error = bus_dmamap_load(buf->dmat, buf->dmap, buf->vaddr,
+ DPAA2_CLASSIFIER_DMA_SIZE, dpaa2_dmamap_oneseg_cb, &buf->paddr,
+ BUS_DMA_NOWAIT);
if (error != 0) {
device_printf(sc->dev, "%s: failed to map a buffer for Rx "
"traffic distribution key configuration\n", __func__);
@@ -4044,8 +3590,7 @@
}
error = DPAA2_CMD_NI_SET_RX_TC_DIST(dev, child, &cmd,
- sc->attr.num.queues, 0, DPAA2_NI_DIST_MODE_HASH,
- buf->store.paddr);
+ sc->attr.num.queues, 0, DPAA2_NI_DIST_MODE_HASH, buf->paddr);
if (error != 0) {
device_printf(dev, "%s: failed to set distribution mode "
"and size for the traffic class\n", __func__);
@@ -4122,36 +3667,6 @@
return (0);
}
-/**
- * @brief Obtain the next dequeue response from the channel storage.
- */
-static int
-dpaa2_ni_chan_storage_next(struct dpaa2_ni_channel *chan, struct dpaa2_dq **dq)
-{
- struct dpaa2_buf *buf = &chan->store;
- struct dpaa2_dq *msgs = buf->store.vaddr;
- struct dpaa2_dq *msg = &msgs[chan->store_idx];
- int rc = EINPROGRESS;
-
- chan->store_idx++;
-
- if (msg->fdr.desc.stat & DPAA2_DQ_STAT_EXPIRED) {
- rc = EALREADY; /* VDQ command is expired */
- chan->store_idx = 0;
- if (!(msg->fdr.desc.stat & DPAA2_DQ_STAT_VALIDFRAME))
- msg = NULL; /* Null response, FD is invalid */
- }
- if (msg->fdr.desc.stat & DPAA2_DQ_STAT_FQEMPTY) {
- rc = ENOENT; /* FQ is empty */
- chan->store_idx = 0;
- }
-
- if (dq != NULL)
- *dq = msg;
-
- return (rc);
-}
-
static device_method_t dpaa2_ni_methods[] = {
/* Device interface */
DEVMETHOD(device_probe, dpaa2_ni_probe),
diff --git a/sys/dev/dpaa2/dpaa2_swp.h b/sys/dev/dpaa2/dpaa2_swp.h
--- a/sys/dev/dpaa2/dpaa2_swp.h
+++ b/sys/dev/dpaa2/dpaa2_swp.h
@@ -33,6 +33,7 @@
#include <sys/bus.h>
#include "dpaa2_types.h"
+#include "dpaa2_buf.h"
#include "dpaa2_bp.h"
/*
@@ -318,16 +319,16 @@
struct dpaa2_fa {
uint32_t magic;
struct dpaa2_buf *buf;
+#ifdef __notyet__
union {
struct { /* Tx frame annotation */
struct dpaa2_ni_tx_ring *tx;
};
-#ifdef __notyet__
struct { /* Rx frame annotation */
uint64_t _notused;
};
-#endif
};
+#endif
} __packed;
CTASSERT(sizeof(struct dpaa2_fa) <= DPAA2_FA_SIZE);
diff --git a/sys/dev/dpaa2/dpaa2_swp.c b/sys/dev/dpaa2/dpaa2_swp.c
--- a/sys/dev/dpaa2/dpaa2_swp.c
+++ b/sys/dev/dpaa2/dpaa2_swp.c
@@ -78,7 +78,6 @@
#include <sys/time.h>
#include <sys/types.h>
#include <sys/systm.h>
-#include <sys/condvar.h>
#include <sys/lock.h>
#include <machine/bus.h>
@@ -298,8 +297,9 @@
& p->eqcr.pi_ci_mask;
p->eqcr.available = p->eqcr.pi_ring_size;
- /* Initialize the portal with an IRQ threshold and timeout of 0us. */
- dpaa2_swp_set_irq_coalescing(p, p->dqrr.ring_size - 1, 0);
+ /* TODO: sysctl(9) for the IRQ timeout? */
+ /* Initialize the portal with an IRQ threshold and timeout of 120us. */
+ dpaa2_swp_set_irq_coalescing(p, p->dqrr.ring_size - 1, 120);
*swp = p;
@@ -724,16 +724,13 @@
uint16_t flags;
int i, error;
- KASSERT(swp != NULL, ("%s: swp is NULL", __func__));
KASSERT(frames_n != 0u, ("%s: cannot pull zero frames", __func__));
KASSERT(frames_n <= 16u, ("%s: too much frames to pull", __func__));
- KASSERT(buf->type == DPAA2_BUF_STORE, ("%s: not channel storage "
- "buffer", __func__));
cmd.numf = frames_n - 1;
cmd.tok = DPAA2_SWP_VDQCR_TOKEN;
cmd.dq_src = chan_id;
- cmd.rsp_addr = (uint64_t) buf->store.paddr;
+ cmd.rsp_addr = (uint64_t)buf->paddr;
/* Dequeue command type */
cmd.verb &= ~(1 << QB_VDQCR_VERB_DCT0_SHIFT);
@@ -763,10 +760,10 @@
}
/* Let's sync before reading VDQ response from QBMan. */
- bus_dmamap_sync(buf->store.dmat, buf->store.dmap, BUS_DMASYNC_POSTREAD);
+ bus_dmamap_sync(buf->dmat, buf->dmap, BUS_DMASYNC_POSTREAD);
/* Read VDQ response from QBMan. */
- msg = (struct dpaa2_dq *) buf->store.vaddr;
+ msg = (struct dpaa2_dq *)buf->vaddr;
for (i = 1; i <= CMD_SPIN_ATTEMPTS; i++) {
if ((msg->fdr.desc.stat & DPAA2_DQ_STAT_VOLATILE) &&
(msg->fdr.desc.tok == DPAA2_SWP_VDQCR_TOKEN)) {
diff --git a/sys/dev/dpaa2/dpaa2_types.h b/sys/dev/dpaa2/dpaa2_types.h
--- a/sys/dev/dpaa2/dpaa2_types.h
+++ b/sys/dev/dpaa2/dpaa2_types.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
- * Copyright © 2021-2022 Dmitry Salychev
+ * Copyright © 2021-2023 Dmitry Salychev
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -28,10 +28,18 @@
#ifndef _DPAA2_TYPES_H
#define _DPAA2_TYPES_H
+#include <sys/param.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+
#include <machine/atomic.h>
+#include <machine/bus.h>
#define DPAA2_MAGIC ((uint32_t) 0xD4AA2C0Du)
+#define DPAA2_MAX_CHANNELS 16 /* CPU cores */
+#define DPAA2_MAX_TCS 8 /* Traffic classes */
+
/**
* @brief Types of the DPAA2 devices.
*/
@@ -51,56 +59,67 @@
};
/**
- * @brief Types of the DPAA2 buffers.
+ * @brief Types of the DPNI queues.
*/
-enum dpaa2_buf_type {
- DPAA2_BUF_RX = 75, /* Rx buffer */
- DPAA2_BUF_TX, /* Tx buffer */
- DPAA2_BUF_STORE /* Channel storage, key configuration */
-};
-
-/**
- * @brief DMA-mapped buffer (for Rx/Tx buffers, channel storage, etc.).
- */
-struct dpaa2_buf {
- enum dpaa2_buf_type type;
- union {
- struct {
- bus_dma_tag_t dmat; /* DMA tag for this buffer */
- bus_dmamap_t dmap;
- bus_addr_t paddr;
- void *vaddr;
-
- struct mbuf *m; /* associated mbuf */
- } rx;
- struct {
- bus_dma_tag_t dmat; /* DMA tag for this buffer */
- bus_dmamap_t dmap;
- bus_addr_t paddr;
- void *vaddr;
-
- struct mbuf *m; /* associated mbuf */
- uint64_t idx;
-
- /* for scatter/gather table */
- bus_dma_tag_t sgt_dmat;
- bus_dmamap_t sgt_dmap;
- bus_addr_t sgt_paddr;
- void *sgt_vaddr;
- } tx;
- struct {
- bus_dma_tag_t dmat; /* DMA tag for this buffer */
- bus_dmamap_t dmap;
- bus_addr_t paddr;
- void *vaddr;
- } store;
- };
+enum dpaa2_ni_queue_type {
+ DPAA2_NI_QUEUE_RX = 0,
+ DPAA2_NI_QUEUE_TX,
+ DPAA2_NI_QUEUE_TX_CONF,
+ DPAA2_NI_QUEUE_RX_ERR
};
struct dpaa2_atomic {
volatile int counter;
};
+/**
+ * @brief Tx ring.
+ *
+ * fq: Parent (TxConf) frame queue.
+ * fqid: ID of the logical Tx queue.
+ * br: Ring buffer for mbufs to transmit.
+ * lock: Lock for the ring buffer.
+ */
+struct dpaa2_ni_tx_ring {
+ struct dpaa2_ni_fq *fq;
+ uint32_t fqid;
+ uint32_t txid; /* Tx ring index */
+
+ struct buf_ring *br;
+ struct mtx lock;
+} __aligned(CACHE_LINE_SIZE);
+
+/**
+ * @brief Frame Queue is the basic queuing structure used by the QMan.
+ *
+ * It comprises a list of frame descriptors (FDs), so it can be thought of
+ * as a queue of frames.
+ *
+ * NOTE: When frames on a FQ are ready to be processed, the FQ is enqueued
+ * onto a work queue (WQ).
+ *
+ * fqid: Frame queue ID, can be used to enqueue/dequeue or execute other
+ * commands on the queue through DPIO.
+ * txq_n: Number of configured Tx queues.
+ * tx_fqid: Frame queue IDs of the Tx queues which belong to the same flowid.
+ * Note that Tx queues are logical queues and not all management
+ * commands are available on these queue types.
+ * qdbin: Queue destination bin. Can be used with the DPIO enqueue
+ * operation based on QDID, QDBIN and QPRI. Note that all Tx queues
+ * with the same flowid have the same destination bin.
+ */
+struct dpaa2_ni_fq {
+ struct dpaa2_channel *chan;
+ uint32_t fqid;
+ uint16_t flowid;
+ uint8_t tc;
+ enum dpaa2_ni_queue_type type;
+
+ /* Optional fields (for TxConf queue). */
+ struct dpaa2_ni_tx_ring tx_rings[DPAA2_MAX_TCS];
+ uint32_t tx_qdbin;
+} __aligned(CACHE_LINE_SIZE);
+
/* Handy wrappers over atomic operations. */
#define DPAA2_ATOMIC_XCHG(a, val) \
(atomic_swap_int(&(a)->counter, (val)))
@@ -109,8 +128,8 @@
#define DPAA2_ATOMIC_ADD(a, val) \
(atomic_add_acq_int(&(a)->counter, (val)))
-/* Convert DPAA2 type to/from string. */
-const char *dpaa2_ttos(enum dpaa2_dev_type type);
-enum dpaa2_dev_type dpaa2_stot(const char *str);
+const char *dpaa2_ttos(enum dpaa2_dev_type);
+enum dpaa2_dev_type dpaa2_stot(const char *);
+void dpaa2_dmamap_oneseg_cb(void *, bus_dma_segment_t *, int, int);
#endif /* _DPAA2_TYPES_H */
diff --git a/sys/dev/dpaa2/dpaa2_types.c b/sys/dev/dpaa2/dpaa2_types.c
new file mode 100644
--- /dev/null
+++ b/sys/dev/dpaa2/dpaa2_types.c
@@ -0,0 +1,114 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright © 2023 Dmitry Salychev
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/types.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+
+#include <machine/bus.h>
+
+#include "dpaa2_types.h"
+
+#define COMPARE_TYPE(t, v) (strncmp((v), (t), strlen((v))) == 0)
+
+/**
+ * @brief Convert DPAA2 device type to string.
+ */
+const char *
+dpaa2_ttos(enum dpaa2_dev_type type)
+{
+ switch (type) {
+ case DPAA2_DEV_MC:
+ return ("mc"); /* NOTE: to print as information only. */
+ case DPAA2_DEV_RC:
+ return ("dprc");
+ case DPAA2_DEV_IO:
+ return ("dpio");
+ case DPAA2_DEV_NI:
+ return ("dpni");
+ case DPAA2_DEV_MCP:
+ return ("dpmcp");
+ case DPAA2_DEV_BP:
+ return ("dpbp");
+ case DPAA2_DEV_CON:
+ return ("dpcon");
+ case DPAA2_DEV_MAC:
+ return ("dpmac");
+ case DPAA2_DEV_MUX:
+ return ("dpdmux");
+ case DPAA2_DEV_SW:
+ return ("dpsw");
+ default:
+ break;
+ }
+
+ return ("notype");
+}
+
+/**
+ * @brief Convert string to DPAA2 device type.
+ */
+enum dpaa2_dev_type
+dpaa2_stot(const char *str)
+{
+ if (COMPARE_TYPE(str, "dprc")) {
+ return (DPAA2_DEV_RC);
+ } else if (COMPARE_TYPE(str, "dpio")) {
+ return (DPAA2_DEV_IO);
+ } else if (COMPARE_TYPE(str, "dpni")) {
+ return (DPAA2_DEV_NI);
+ } else if (COMPARE_TYPE(str, "dpmcp")) {
+ return (DPAA2_DEV_MCP);
+ } else if (COMPARE_TYPE(str, "dpbp")) {
+ return (DPAA2_DEV_BP);
+ } else if (COMPARE_TYPE(str, "dpcon")) {
+ return (DPAA2_DEV_CON);
+ } else if (COMPARE_TYPE(str, "dpmac")) {
+ return (DPAA2_DEV_MAC);
+ } else if (COMPARE_TYPE(str, "dpdmux")) {
+ return (DPAA2_DEV_MUX);
+ } else if (COMPARE_TYPE(str, "dpsw")) {
+ return (DPAA2_DEV_SW);
+ }
+
+ return (DPAA2_DEV_NOTYPE);
+}
+
+/**
+ * @brief Callback to obtain a physical address of the only DMA segment mapped.
+ */
+void
+dpaa2_dmamap_oneseg_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
+{
+ if (error == 0) {
+ KASSERT(nseg == 1, ("%s: too many segments: nseg=%d\n",
+ __func__, nseg));
+ *(bus_addr_t *)arg = segs[0].ds_addr;
+ } else {
+ panic("%s: error=%d\n", __func__, error);
+ }
+}
diff --git a/sys/modules/dpaa2/Makefile b/sys/modules/dpaa2/Makefile
--- a/sys/modules/dpaa2/Makefile
+++ b/sys/modules/dpaa2/Makefile
@@ -11,6 +11,9 @@
SRCS+= dpaa2_swp.c
SRCS+= dpaa2_mac.c
SRCS+= dpaa2_con.c
+SRCS+= dpaa2_buf.c
+SRCS+= dpaa2_channel.c
+SRCS+= dpaa2_types.c
SRCS+= dpaa2_cmd_if.c dpaa2_cmd_if.h
SRCS+= dpaa2_swp_if.c dpaa2_swp_if.h
@@ -21,12 +24,15 @@
SRCS+= bus_if.h device_if.h miibus_if.h
SRCS+= pcib_if.h pci_if.h
-SRCS+= opt_acpi.h opt_platform.h
+
+SRCS+= opt_acpi.h
+SRCS+= opt_platform.h
+SRCS+= opt_rss.h
SRCS.DEV_ACPI= dpaa2_mc_acpi.c \
- memac_mdio_acpi.c \
- acpi_if.h \
- acpi_bus_if.h
+ memac_mdio_acpi.c \
+ acpi_if.h \
+ acpi_bus_if.h
.if !empty(OPT_FDT)
SRCS+= dpaa2_mc_fdt.c \

File Metadata

Mime Type
text/plain
Expires
Sat, Feb 15, 9:22 PM (13 h, 19 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
16665071
Default Alt Text
D41296.diff (125 KB)

Event Timeline