Command Push Pull is the name of NFP's network on a chip.
PCIe PF can access the interconnect through a number of mappings
controlled via Base Access Registers. BARs allow the PF to issue
pretty much any command or address any memory on the chip.
Add appropriate logic and a handful of helper for simple operations
like reading scalars from memories.
Signed-off-by: Jakub Kicinski <jakub.kicinski@netronome.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
obj-$(CONFIG_NFP) += nfp.o
nfp-objs := \
+ nfpcore/nfp6000_pcie.o \
+ nfpcore/nfp_cppcore.o \
+ nfpcore/nfp_cpplib.o \
+ nfpcore/nfp_target.o \
nfp_main.o \
nfp_net_common.o \
nfp_net_ethtool.o \
--- /dev/null
+/*
+ * Copyright (C) 2015-2017 Netronome Systems, Inc.
+ *
+ * This software is dual licensed under the GNU General License Version 2,
+ * June 1991 as shown in the file COPYING in the top-level directory of this
+ * source tree or the BSD 2-Clause License provided below. You have the
+ * option to license this software under the complete terms of either license.
+ *
+ * The BSD 2-Clause License:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef NFP6000_NFP6000_H
+#define NFP6000_NFP6000_H
+
+#include <linux/errno.h>
+#include <linux/types.h>
+
+/* CPP Target IDs */
+#define NFP_CPP_TARGET_INVALID 0
+#define NFP_CPP_TARGET_NBI 1
+#define NFP_CPP_TARGET_QDR 2
+#define NFP_CPP_TARGET_ILA 6
+#define NFP_CPP_TARGET_MU 7
+#define NFP_CPP_TARGET_PCIE 9
+#define NFP_CPP_TARGET_ARM 10
+#define NFP_CPP_TARGET_CRYPTO 12
+#define NFP_CPP_TARGET_ISLAND_XPB 14 /* Shared with CAP */
+#define NFP_CPP_TARGET_ISLAND_CAP 14 /* Shared with XPB */
+#define NFP_CPP_TARGET_CT_XPB 14
+#define NFP_CPP_TARGET_LOCAL_SCRATCH 15
+#define NFP_CPP_TARGET_CLS NFP_CPP_TARGET_LOCAL_SCRATCH
+
+#define NFP_ISL_EMEM0 24
+
+#define NFP_MU_ADDR_ACCESS_TYPE_MASK 3ULL
+#define NFP_MU_ADDR_ACCESS_TYPE_DIRECT 2ULL
+
+#define PUSHPULL(_pull, _push) ((_pull) << 4 | (_push) << 0)
+#define PUSH_WIDTH(_pushpull) pushpull_width((_pushpull) >> 0)
+#define PULL_WIDTH(_pushpull) pushpull_width((_pushpull) >> 4)
+
+static inline int pushpull_width(int pp)
+{
+ pp &= 0xf;
+
+ if (pp == 0)
+ return -EINVAL;
+ return 2 << pp;
+}
+
+static inline int nfp_cppat_mu_locality_lsb(int mode, bool addr40)
+{
+ switch (mode) {
+ case 0 ... 3:
+ return addr40 ? 38 : 30;
+ default:
+ return -EINVAL;
+ }
+}
+
+int nfp_target_pushpull(u32 cpp_id, u64 address);
+int nfp_target_cpp(u32 cpp_island_id, u64 cpp_island_address,
+ u32 *cpp_target_id, u64 *cpp_target_address,
+ const u32 *imb_table);
+
+#endif /* NFP6000_NFP6000_H */
--- /dev/null
+/*
+ * Copyright (C) 2015-2017 Netronome Systems, Inc.
+ *
+ * This software is dual licensed under the GNU General License Version 2,
+ * June 1991 as shown in the file COPYING in the top-level directory of this
+ * source tree or the BSD 2-Clause License provided below. You have the
+ * option to license this software under the complete terms of either license.
+ *
+ * The BSD 2-Clause License:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+/*
+ * nfp_xpb.h
+ * Author: Jason McMullan <jason.mcmullan@netronome.com>
+ */
+
+#ifndef NFP6000_XPB_H
+#define NFP6000_XPB_H
+
+/* For use with NFP6000 Databook "XPB Addressing" section
+ */
+#define NFP_XPB_OVERLAY(island) (((island) & 0x3f) << 24)
+
+#define NFP_XPB_ISLAND(island) (NFP_XPB_OVERLAY(island) + 0x60000)
+
+#define NFP_XPB_ISLAND_of(offset) (((offset) >> 24) & 0x3F)
+
+/* For use with NFP6000 Databook "XPB Island and Device IDs" chapter
+ */
+#define NFP_XPB_DEVICE(island, slave, device) \
+ (NFP_XPB_OVERLAY(island) | \
+ (((slave) & 3) << 22) | \
+ (((device) & 0x3f) << 16))
+
+#endif /* NFP6000_XPB_H */
--- /dev/null
+/*
+ * Copyright (C) 2015-2017 Netronome Systems, Inc.
+ *
+ * This software is dual licensed under the GNU General License Version 2,
+ * June 1991 as shown in the file COPYING in the top-level directory of this
+ * source tree or the BSD 2-Clause License provided below. You have the
+ * option to license this software under the complete terms of either license.
+ *
+ * The BSD 2-Clause License:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+/*
+ * nfp6000_pcie.c
+ * Authors: Jakub Kicinski <jakub.kicinski@netronome.com>
+ * Jason McMullan <jason.mcmullan@netronome.com>
+ * Rolf Neugebauer <rolf.neugebauer@netronome.com>
+ *
+ * Multiplexes the NFP BARs between NFP internal resources and
+ * implements the PCIe specific interface for generic CPP bus access.
+ *
+ * The BARs are managed with refcounts and are allocated/acquired
+ * using target, token and offset/size matching. The generic CPP bus
+ * abstraction builds upon this BAR interface.
+ */
+
+#include <asm/unaligned.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/kref.h>
+#include <linux/io.h>
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/sort.h>
+#include <linux/sched.h>
+#include <linux/types.h>
+#include <linux/pci.h>
+
+#include "nfp_cpp.h"
+
+#include "nfp6000/nfp6000.h"
+
+#include "nfp6000_pcie.h"
+
+#define NFP_PCIE_BAR(_pf) (0x30000 + ((_pf) & 7) * 0xc0)
+#define NFP_PCIE_BAR_EXPLICIT_BAR0(_x, _y) \
+ (0x00000080 + (0x40 * ((_x) & 0x3)) + (0x10 * ((_y) & 0x3)))
+#define NFP_PCIE_BAR_EXPLICIT_BAR0_SignalType(_x) (((_x) & 0x3) << 30)
+#define NFP_PCIE_BAR_EXPLICIT_BAR0_SignalType_of(_x) (((_x) >> 30) & 0x3)
+#define NFP_PCIE_BAR_EXPLICIT_BAR0_Token(_x) (((_x) & 0x3) << 28)
+#define NFP_PCIE_BAR_EXPLICIT_BAR0_Token_of(_x) (((_x) >> 28) & 0x3)
+#define NFP_PCIE_BAR_EXPLICIT_BAR0_Address(_x) (((_x) & 0xffffff) << 0)
+#define NFP_PCIE_BAR_EXPLICIT_BAR0_Address_of(_x) (((_x) >> 0) & 0xffffff)
+#define NFP_PCIE_BAR_EXPLICIT_BAR1(_x, _y) \
+ (0x00000084 + (0x40 * ((_x) & 0x3)) + (0x10 * ((_y) & 0x3)))
+#define NFP_PCIE_BAR_EXPLICIT_BAR1_SignalRef(_x) (((_x) & 0x7f) << 24)
+#define NFP_PCIE_BAR_EXPLICIT_BAR1_SignalRef_of(_x) (((_x) >> 24) & 0x7f)
+#define NFP_PCIE_BAR_EXPLICIT_BAR1_DataMaster(_x) (((_x) & 0x3ff) << 14)
+#define NFP_PCIE_BAR_EXPLICIT_BAR1_DataMaster_of(_x) (((_x) >> 14) & 0x3ff)
+#define NFP_PCIE_BAR_EXPLICIT_BAR1_DataRef(_x) (((_x) & 0x3fff) << 0)
+#define NFP_PCIE_BAR_EXPLICIT_BAR1_DataRef_of(_x) (((_x) >> 0) & 0x3fff)
+#define NFP_PCIE_BAR_EXPLICIT_BAR2(_x, _y) \
+ (0x00000088 + (0x40 * ((_x) & 0x3)) + (0x10 * ((_y) & 0x3)))
+#define NFP_PCIE_BAR_EXPLICIT_BAR2_Target(_x) (((_x) & 0xf) << 28)
+#define NFP_PCIE_BAR_EXPLICIT_BAR2_Target_of(_x) (((_x) >> 28) & 0xf)
+#define NFP_PCIE_BAR_EXPLICIT_BAR2_Action(_x) (((_x) & 0x1f) << 23)
+#define NFP_PCIE_BAR_EXPLICIT_BAR2_Action_of(_x) (((_x) >> 23) & 0x1f)
+#define NFP_PCIE_BAR_EXPLICIT_BAR2_Length(_x) (((_x) & 0x1f) << 18)
+#define NFP_PCIE_BAR_EXPLICIT_BAR2_Length_of(_x) (((_x) >> 18) & 0x1f)
+#define NFP_PCIE_BAR_EXPLICIT_BAR2_ByteMask(_x) (((_x) & 0xff) << 10)
+#define NFP_PCIE_BAR_EXPLICIT_BAR2_ByteMask_of(_x) (((_x) >> 10) & 0xff)
+#define NFP_PCIE_BAR_EXPLICIT_BAR2_SignalMaster(_x) (((_x) & 0x3ff) << 0)
+#define NFP_PCIE_BAR_EXPLICIT_BAR2_SignalMaster_of(_x) (((_x) >> 0) & 0x3ff)
+
+#define NFP_PCIE_BAR_PCIE2CPP_Action_BaseAddress(_x) (((_x) & 0x1f) << 16)
+#define NFP_PCIE_BAR_PCIE2CPP_Action_BaseAddress_of(_x) (((_x) >> 16) & 0x1f)
+#define NFP_PCIE_BAR_PCIE2CPP_BaseAddress(_x) (((_x) & 0xffff) << 0)
+#define NFP_PCIE_BAR_PCIE2CPP_BaseAddress_of(_x) (((_x) >> 0) & 0xffff)
+#define NFP_PCIE_BAR_PCIE2CPP_LengthSelect(_x) (((_x) & 0x3) << 27)
+#define NFP_PCIE_BAR_PCIE2CPP_LengthSelect_of(_x) (((_x) >> 27) & 0x3)
+#define NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT 0
+#define NFP_PCIE_BAR_PCIE2CPP_LengthSelect_64BIT 1
+#define NFP_PCIE_BAR_PCIE2CPP_LengthSelect_0BYTE 3
+#define NFP_PCIE_BAR_PCIE2CPP_MapType(_x) (((_x) & 0x7) << 29)
+#define NFP_PCIE_BAR_PCIE2CPP_MapType_of(_x) (((_x) >> 29) & 0x7)
+#define NFP_PCIE_BAR_PCIE2CPP_MapType_FIXED 0
+#define NFP_PCIE_BAR_PCIE2CPP_MapType_BULK 1
+#define NFP_PCIE_BAR_PCIE2CPP_MapType_TARGET 2
+#define NFP_PCIE_BAR_PCIE2CPP_MapType_GENERAL 3
+#define NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT0 4
+#define NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT1 5
+#define NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT2 6
+#define NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT3 7
+#define NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress(_x) (((_x) & 0xf) << 23)
+#define NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress_of(_x) (((_x) >> 23) & 0xf)
+#define NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress(_x) (((_x) & 0x3) << 21)
+#define NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress_of(_x) (((_x) >> 21) & 0x3)
+#define NFP_PCIE_EM 0x020000
+#define NFP_PCIE_SRAM 0x000000
+
+#define NFP_PCIE_P2C_FIXED_SIZE(bar) (1 << (bar)->bitsize)
+#define NFP_PCIE_P2C_BULK_SIZE(bar) (1 << (bar)->bitsize)
+#define NFP_PCIE_P2C_GENERAL_TARGET_OFFSET(bar, x) ((x) << ((bar)->bitsize - 2))
+#define NFP_PCIE_P2C_GENERAL_TOKEN_OFFSET(bar, x) ((x) << ((bar)->bitsize - 4))
+#define NFP_PCIE_P2C_GENERAL_SIZE(bar) (1 << ((bar)->bitsize - 4))
+
+#define NFP_PCIE_CFG_BAR_PCIETOCPPEXPANSIONBAR(bar, slot) \
+ (0x400 + ((bar) * 8 + (slot)) * 4)
+
+#define NFP_PCIE_CPP_BAR_PCIETOCPPEXPANSIONBAR(bar, slot) \
+ (((bar) * 8 + (slot)) * 4)
+
+/* The number of explicit BARs to reserve.
+ * Minimum is 0, maximum is 4 on the NFP6000.
+ */
+#define NFP_PCIE_EXPLICIT_BARS 2
+
+struct nfp6000_pcie;
+struct nfp6000_area_priv;
+
+/**
+ * struct nfp_bar - describes BAR configuration and usage
+ * @nfp: backlink to owner
+ * @barcfg: cached contents of BAR config CSR
+ * @base: the BAR's base CPP offset
+ * @mask: mask for the BAR aperture (read only)
+ * @bitsize: bitsize of BAR aperture (read only)
+ * @index: index of the BAR
+ * @refcnt: number of current users
+ * @iomem: mapped IO memory
+ * @resource: iomem resource window
+ */
+struct nfp_bar {
+ struct nfp6000_pcie *nfp;
+ u32 barcfg;
+ u64 base; /* CPP address base */
+ u64 mask; /* Bit mask of the bar */
+ u32 bitsize; /* Bit size of the bar */
+ int index;
+ atomic_t refcnt;
+
+ void __iomem *iomem;
+ struct resource *resource;
+};
+
+#define NFP_PCI_BAR_MAX (PCI_64BIT_BAR_COUNT * 8)
+
+struct nfp6000_pcie {
+ struct pci_dev *pdev;
+ struct device *dev;
+
+ /* PCI BAR management */
+ spinlock_t bar_lock; /* Protect the PCI2CPP BAR cache */
+ int bars;
+ struct nfp_bar bar[NFP_PCI_BAR_MAX];
+ wait_queue_head_t bar_waiters;
+
+ /* Reserved BAR access */
+ struct {
+ void __iomem *csr;
+ void __iomem *em;
+ void __iomem *expl[4];
+ } iomem;
+
+ /* Explicit IO access */
+ struct {
+ struct mutex mutex; /* Lock access to this explicit group */
+ u8 master_id;
+ u8 signal_ref;
+ void __iomem *data;
+ struct {
+ void __iomem *addr;
+ int bitsize;
+ int free[4];
+ } group[4];
+ } expl;
+};
+
+static u32 nfp_bar_maptype(struct nfp_bar *bar)
+{
+ return NFP_PCIE_BAR_PCIE2CPP_MapType_of(bar->barcfg);
+}
+
+static resource_size_t nfp_bar_resource_len(struct nfp_bar *bar)
+{
+ return pci_resource_len(bar->nfp->pdev, (bar->index / 8) * 2) / 8;
+}
+
+static resource_size_t nfp_bar_resource_start(struct nfp_bar *bar)
+{
+ return pci_resource_start(bar->nfp->pdev, (bar->index / 8) * 2)
+ + nfp_bar_resource_len(bar) * (bar->index & 7);
+}
+
+#define TARGET_WIDTH_32 4
+#define TARGET_WIDTH_64 8
+
+static int
+compute_bar(struct nfp6000_pcie *nfp, struct nfp_bar *bar,
+ u32 *bar_config, u64 *bar_base,
+ int tgt, int act, int tok, u64 offset, size_t size, int width)
+{
+ int bitsize;
+ u32 newcfg;
+
+ if (tgt >= NFP_CPP_NUM_TARGETS)
+ return -EINVAL;
+
+ switch (width) {
+ case 8:
+ newcfg = NFP_PCIE_BAR_PCIE2CPP_LengthSelect(
+ NFP_PCIE_BAR_PCIE2CPP_LengthSelect_64BIT);
+ break;
+ case 4:
+ newcfg = NFP_PCIE_BAR_PCIE2CPP_LengthSelect(
+ NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT);
+ break;
+ case 0:
+ newcfg = NFP_PCIE_BAR_PCIE2CPP_LengthSelect(
+ NFP_PCIE_BAR_PCIE2CPP_LengthSelect_0BYTE);
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ if (act != NFP_CPP_ACTION_RW && act != 0) {
+ /* Fixed CPP mapping with specific action */
+ u64 mask = ~(NFP_PCIE_P2C_FIXED_SIZE(bar) - 1);
+
+ newcfg |= NFP_PCIE_BAR_PCIE2CPP_MapType(
+ NFP_PCIE_BAR_PCIE2CPP_MapType_FIXED);
+ newcfg |= NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress(tgt);
+ newcfg |= NFP_PCIE_BAR_PCIE2CPP_Action_BaseAddress(act);
+ newcfg |= NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress(tok);
+
+ if ((offset & mask) != ((offset + size - 1) & mask))
+ return -EINVAL;
+ offset &= mask;
+
+ bitsize = 40 - 16;
+ } else {
+ u64 mask = ~(NFP_PCIE_P2C_BULK_SIZE(bar) - 1);
+
+ /* Bulk mapping */
+ newcfg |= NFP_PCIE_BAR_PCIE2CPP_MapType(
+ NFP_PCIE_BAR_PCIE2CPP_MapType_BULK);
+ newcfg |= NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress(tgt);
+ newcfg |= NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress(tok);
+
+ if ((offset & mask) != ((offset + size - 1) & mask))
+ return -EINVAL;
+
+ offset &= mask;
+
+ bitsize = 40 - 21;
+ }
+
+ if (bar->bitsize < bitsize)
+ return -EINVAL;
+
+ newcfg |= offset >> bitsize;
+
+ if (bar_base)
+ *bar_base = offset;
+
+ if (bar_config)
+ *bar_config = newcfg;
+
+ return 0;
+}
+
+static int
+nfp6000_bar_write(struct nfp6000_pcie *nfp, struct nfp_bar *bar, u32 newcfg)
+{
+ int base, slot;
+ int xbar;
+
+ base = bar->index >> 3;
+ slot = bar->index & 7;
+
+ if (nfp->iomem.csr) {
+ xbar = NFP_PCIE_CPP_BAR_PCIETOCPPEXPANSIONBAR(base, slot);
+ writel(newcfg, nfp->iomem.csr + xbar);
+ /* Readback to ensure BAR is flushed */
+ readl(nfp->iomem.csr + xbar);
+ } else {
+ xbar = NFP_PCIE_CFG_BAR_PCIETOCPPEXPANSIONBAR(base, slot);
+ pci_write_config_dword(nfp->pdev, xbar, newcfg);
+ }
+
+ bar->barcfg = newcfg;
+
+ return 0;
+}
+
+static int
+reconfigure_bar(struct nfp6000_pcie *nfp, struct nfp_bar *bar,
+ int tgt, int act, int tok, u64 offset, size_t size, int width)
+{
+ u64 newbase;
+ u32 newcfg;
+ int err;
+
+ err = compute_bar(nfp, bar, &newcfg, &newbase,
+ tgt, act, tok, offset, size, width);
+ if (err)
+ return err;
+
+ bar->base = newbase;
+
+ return nfp6000_bar_write(nfp, bar, newcfg);
+}
+
+/* Check if BAR can be used with the given parameters. */
+static int matching_bar(struct nfp_bar *bar, u32 tgt, u32 act, u32 tok,
+ u64 offset, size_t size, int width)
+{
+ int bartgt, baract, bartok;
+ int barwidth;
+ u32 maptype;
+
+ maptype = NFP_PCIE_BAR_PCIE2CPP_MapType_of(bar->barcfg);
+ bartgt = NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress_of(bar->barcfg);
+ bartok = NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress_of(bar->barcfg);
+ baract = NFP_PCIE_BAR_PCIE2CPP_Action_BaseAddress_of(bar->barcfg);
+
+ barwidth = NFP_PCIE_BAR_PCIE2CPP_LengthSelect_of(bar->barcfg);
+ switch (barwidth) {
+ case NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT:
+ barwidth = 4;
+ break;
+ case NFP_PCIE_BAR_PCIE2CPP_LengthSelect_64BIT:
+ barwidth = 8;
+ break;
+ case NFP_PCIE_BAR_PCIE2CPP_LengthSelect_0BYTE:
+ barwidth = 0;
+ break;
+ default:
+ barwidth = -1;
+ break;
+ }
+
+ switch (maptype) {
+ case NFP_PCIE_BAR_PCIE2CPP_MapType_TARGET:
+ bartok = -1;
+ /* FALLTHROUGH */
+ case NFP_PCIE_BAR_PCIE2CPP_MapType_BULK:
+ baract = NFP_CPP_ACTION_RW;
+ if (act == 0)
+ act = NFP_CPP_ACTION_RW;
+ /* FALLTHROUGH */
+ case NFP_PCIE_BAR_PCIE2CPP_MapType_FIXED:
+ break;
+ default:
+ /* We don't match explicit bars through the area interface */
+ return 0;
+ }
+
+ /* Make sure to match up the width */
+ if (barwidth != width)
+ return 0;
+
+ if ((bartgt < 0 || bartgt == tgt) &&
+ (bartok < 0 || bartok == tok) &&
+ (baract == act) &&
+ bar->base <= offset &&
+ (bar->base + (1 << bar->bitsize)) >= (offset + size))
+ return 1;
+
+ /* No match */
+ return 0;
+}
+
+static int
+find_matching_bar(struct nfp6000_pcie *nfp,
+ u32 tgt, u32 act, u32 tok, u64 offset, size_t size, int width)
+{
+ int n;
+
+ for (n = 0; n < nfp->bars; n++) {
+ struct nfp_bar *bar = &nfp->bar[n];
+
+ if (matching_bar(bar, tgt, act, tok, offset, size, width))
+ return n;
+ }
+
+ return -1;
+}
+
+/* Return EAGAIN if no resource is available */
+static int
+find_unused_bar_noblock(struct nfp6000_pcie *nfp,
+ int tgt, int act, int tok,
+ u64 offset, size_t size, int width)
+{
+ int n, invalid = 0;
+
+ for (n = 0; n < nfp->bars; n++) {
+ struct nfp_bar *bar = &nfp->bar[n];
+ int err;
+
+ if (bar->bitsize == 0) {
+ invalid++;
+ continue;
+ }
+
+ if (atomic_read(&bar->refcnt) != 0)
+ continue;
+
+ /* Just check to see if we can make it fit... */
+ err = compute_bar(nfp, bar, NULL, NULL,
+ tgt, act, tok, offset, size, width);
+
+ if (err < 0)
+ invalid++;
+ else
+ return n;
+ }
+
+ return (n == invalid) ? -EINVAL : -EAGAIN;
+}
+
+static int
+find_unused_bar_and_lock(struct nfp6000_pcie *nfp,
+ int tgt, int act, int tok,
+ u64 offset, size_t size, int width)
+{
+ unsigned long flags;
+ int n;
+
+ spin_lock_irqsave(&nfp->bar_lock, flags);
+
+ n = find_unused_bar_noblock(nfp, tgt, act, tok, offset, size, width);
+ if (n < 0)
+ spin_unlock_irqrestore(&nfp->bar_lock, flags);
+ else
+ __release(&nfp->bar_lock);
+
+ return n;
+}
+
+static void nfp_bar_get(struct nfp6000_pcie *nfp, struct nfp_bar *bar)
+{
+ atomic_inc(&bar->refcnt);
+}
+
+static void nfp_bar_put(struct nfp6000_pcie *nfp, struct nfp_bar *bar)
+{
+ if (atomic_dec_and_test(&bar->refcnt))
+ wake_up_interruptible(&nfp->bar_waiters);
+}
+
+static int
+nfp_wait_for_bar(struct nfp6000_pcie *nfp, int *barnum,
+ u32 tgt, u32 act, u32 tok, u64 offset, size_t size, int width)
+{
+ return wait_event_interruptible(nfp->bar_waiters,
+ (*barnum = find_unused_bar_and_lock(nfp, tgt, act, tok,
+ offset, size, width))
+ != -EAGAIN);
+}
+
+static int
+nfp_alloc_bar(struct nfp6000_pcie *nfp,
+ u32 tgt, u32 act, u32 tok,
+ u64 offset, size_t size, int width, int nonblocking)
+{
+ unsigned long irqflags;
+ int barnum, retval;
+
+ if (size > (1 << 24))
+ return -EINVAL;
+
+ spin_lock_irqsave(&nfp->bar_lock, irqflags);
+ barnum = find_matching_bar(nfp, tgt, act, tok, offset, size, width);
+ if (barnum >= 0) {
+ /* Found a perfect match. */
+ nfp_bar_get(nfp, &nfp->bar[barnum]);
+ spin_unlock_irqrestore(&nfp->bar_lock, irqflags);
+ return barnum;
+ }
+
+ barnum = find_unused_bar_noblock(nfp, tgt, act, tok,
+ offset, size, width);
+ if (barnum < 0) {
+ if (nonblocking)
+ goto err_nobar;
+
+ /* Wait until a BAR becomes available. The
+ * find_unused_bar function will reclaim the bar_lock
+ * if a free BAR is found.
+ */
+ spin_unlock_irqrestore(&nfp->bar_lock, irqflags);
+ retval = nfp_wait_for_bar(nfp, &barnum, tgt, act, tok,
+ offset, size, width);
+ if (retval)
+ return retval;
+ __acquire(&nfp->bar_lock);
+ }
+
+ nfp_bar_get(nfp, &nfp->bar[barnum]);
+ retval = reconfigure_bar(nfp, &nfp->bar[barnum],
+ tgt, act, tok, offset, size, width);
+ if (retval < 0) {
+ nfp_bar_put(nfp, &nfp->bar[barnum]);
+ barnum = retval;
+ }
+
+err_nobar:
+ spin_unlock_irqrestore(&nfp->bar_lock, irqflags);
+ return barnum;
+}
+
+static void disable_bars(struct nfp6000_pcie *nfp);
+
+static int bar_cmp(const void *aptr, const void *bptr)
+{
+ const struct nfp_bar *a = aptr, *b = bptr;
+
+ if (a->bitsize == b->bitsize)
+ return a->index - b->index;
+ else
+ return a->bitsize - b->bitsize;
+}
+
+/* Map all PCI bars and fetch the actual BAR configurations from the
+ * board. We assume that the BAR with the PCIe config block is
+ * already mapped.
+ *
+ * BAR0.0: Reserved for General Mapping (for MSI-X access to PCIe SRAM)
+ * BAR0.1: Reserved for XPB access (for MSI-X access to PCIe PBA)
+ * BAR0.2: --
+ * BAR0.3: --
+ * BAR0.4: Reserved for Explicit 0.0-0.3 access
+ * BAR0.5: Reserved for Explicit 1.0-1.3 access
+ * BAR0.6: Reserved for Explicit 2.0-2.3 access
+ * BAR0.7: Reserved for Explicit 3.0-3.3 access
+ *
+ * BAR1.0-BAR1.7: --
+ * BAR2.0-BAR2.7: --
+ */
+static int enable_bars(struct nfp6000_pcie *nfp, u16 interface)
+{
+ const u32 barcfg_msix_general =
+ NFP_PCIE_BAR_PCIE2CPP_MapType(
+ NFP_PCIE_BAR_PCIE2CPP_MapType_GENERAL) |
+ NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT;
+ const u32 barcfg_msix_xpb =
+ NFP_PCIE_BAR_PCIE2CPP_MapType(
+ NFP_PCIE_BAR_PCIE2CPP_MapType_BULK) |
+ NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT |
+ NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress(
+ NFP_CPP_TARGET_ISLAND_XPB);
+ const u32 barcfg_explicit[4] = {
+ NFP_PCIE_BAR_PCIE2CPP_MapType(
+ NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT0),
+ NFP_PCIE_BAR_PCIE2CPP_MapType(
+ NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT1),
+ NFP_PCIE_BAR_PCIE2CPP_MapType(
+ NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT2),
+ NFP_PCIE_BAR_PCIE2CPP_MapType(
+ NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT3),
+ };
+ struct nfp_bar *bar;
+ int i, bars_free;
+ int expl_groups;
+
+ bar = &nfp->bar[0];
+ for (i = 0; i < ARRAY_SIZE(nfp->bar); i++, bar++) {
+ struct resource *res;
+
+ res = &nfp->pdev->resource[(i >> 3) * 2];
+
+ /* Skip over BARs that are not IORESOURCE_MEM */
+ if (!(resource_type(res) & IORESOURCE_MEM)) {
+ bar--;
+ continue;
+ }
+
+ bar->resource = res;
+ bar->barcfg = 0;
+
+ bar->nfp = nfp;
+ bar->index = i;
+ bar->mask = nfp_bar_resource_len(bar) - 1;
+ bar->bitsize = fls(bar->mask);
+ bar->base = 0;
+ bar->iomem = NULL;
+ }
+
+ nfp->bars = bar - &nfp->bar[0];
+ if (nfp->bars < 8) {
+ dev_err(nfp->dev, "No usable BARs found!\n");
+ return -EINVAL;
+ }
+
+ bars_free = nfp->bars;
+
+ /* Convert unit ID (0..3) to signal master/data master ID (0x40..0x70)
+ */
+ mutex_init(&nfp->expl.mutex);
+
+ nfp->expl.master_id = ((NFP_CPP_INTERFACE_UNIT_of(interface) & 3) + 4)
+ << 4;
+ nfp->expl.signal_ref = 0x10;
+
+ /* Configure, and lock, BAR0.0 for General Target use (MSI-X SRAM) */
+ bar = &nfp->bar[0];
+ bar->iomem = ioremap_nocache(nfp_bar_resource_start(bar),
+ nfp_bar_resource_len(bar));
+ if (bar->iomem) {
+ dev_info(nfp->dev,
+ "BAR0.0 RESERVED: General Mapping/MSI-X SRAM\n");
+ atomic_inc(&bar->refcnt);
+ bars_free--;
+
+ nfp6000_bar_write(nfp, bar, barcfg_msix_general);
+
+ nfp->expl.data = bar->iomem + NFP_PCIE_SRAM + 0x1000;
+ }
+
+ if (nfp->pdev->device == PCI_DEVICE_NFP4000 ||
+ nfp->pdev->device == PCI_DEVICE_NFP6000) {
+ nfp->iomem.csr = bar->iomem + NFP_PCIE_BAR(0);
+ expl_groups = 4;
+ } else {
+ int pf = nfp->pdev->devfn & 7;
+
+ nfp->iomem.csr = bar->iomem + NFP_PCIE_BAR(pf);
+ expl_groups = 1;
+ }
+ nfp->iomem.em = bar->iomem + NFP_PCIE_EM;
+
+ /* Configure, and lock, BAR0.1 for PCIe XPB (MSI-X PBA) */
+ bar = &nfp->bar[1];
+ dev_info(nfp->dev, "BAR0.1 RESERVED: PCIe XPB/MSI-X PBA\n");
+ atomic_inc(&bar->refcnt);
+ bars_free--;
+
+ nfp6000_bar_write(nfp, bar, barcfg_msix_xpb);
+
+ /* Use BAR0.4..BAR0.7 for EXPL IO */
+ for (i = 0; i < 4; i++) {
+ int j;
+
+ if (i >= NFP_PCIE_EXPLICIT_BARS || i >= expl_groups) {
+ nfp->expl.group[i].bitsize = 0;
+ continue;
+ }
+
+ bar = &nfp->bar[4 + i];
+ bar->iomem = ioremap_nocache(nfp_bar_resource_start(bar),
+ nfp_bar_resource_len(bar));
+ if (bar->iomem) {
+ dev_info(nfp->dev,
+ "BAR0.%d RESERVED: Explicit%d Mapping\n",
+ 4 + i, i);
+ atomic_inc(&bar->refcnt);
+ bars_free--;
+
+ nfp->expl.group[i].bitsize = bar->bitsize;
+ nfp->expl.group[i].addr = bar->iomem;
+ nfp6000_bar_write(nfp, bar, barcfg_explicit[i]);
+
+ for (j = 0; j < 4; j++)
+ nfp->expl.group[i].free[j] = true;
+ }
+ nfp->iomem.expl[i] = bar->iomem;
+ }
+
+ /* Sort bars by bit size - use the smallest possible first. */
+ sort(&nfp->bar[0], nfp->bars, sizeof(nfp->bar[0]),
+ bar_cmp, NULL);
+
+ dev_info(nfp->dev, "%d NFP PCI2CPP BARs, %d free\n",
+ nfp->bars, bars_free);
+
+ return 0;
+}
+
+static void disable_bars(struct nfp6000_pcie *nfp)
+{
+ struct nfp_bar *bar = &nfp->bar[0];
+ int n;
+
+ for (n = 0; n < nfp->bars; n++, bar++) {
+ if (bar->iomem) {
+ iounmap(bar->iomem);
+ bar->iomem = NULL;
+ }
+ }
+}
+
+/*
+ * Generic CPP bus access interface.
+ */
+
+struct nfp6000_area_priv {
+ atomic_t refcnt;
+
+ struct nfp_bar *bar;
+ u32 bar_offset;
+
+ u32 target;
+ u32 action;
+ u32 token;
+ u64 offset;
+ struct {
+ int read;
+ int write;
+ int bar;
+ } width;
+ size_t size;
+
+ void __iomem *iomem;
+ phys_addr_t phys;
+ struct resource resource;
+};
+
+static int nfp6000_area_init(struct nfp_cpp_area *area, u32 dest,
+ unsigned long long address, unsigned long size)
+{
+ struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
+ u32 target = NFP_CPP_ID_TARGET_of(dest);
+ u32 action = NFP_CPP_ID_ACTION_of(dest);
+ u32 token = NFP_CPP_ID_TOKEN_of(dest);
+ int pp;
+
+ pp = nfp_target_pushpull(NFP_CPP_ID(target, action, token), address);
+ if (pp < 0)
+ return pp;
+
+ priv->width.read = PUSH_WIDTH(pp);
+ priv->width.write = PULL_WIDTH(pp);
+ if (priv->width.read > 0 &&
+ priv->width.write > 0 &&
+ priv->width.read != priv->width.write) {
+ return -EINVAL;
+ }
+
+ if (priv->width.read > 0)
+ priv->width.bar = priv->width.read;
+ else
+ priv->width.bar = priv->width.write;
+
+ atomic_set(&priv->refcnt, 0);
+ priv->bar = NULL;
+
+ priv->target = target;
+ priv->action = action;
+ priv->token = token;
+ priv->offset = address;
+ priv->size = size;
+ memset(&priv->resource, 0, sizeof(priv->resource));
+
+ return 0;
+}
+
+static void nfp6000_area_cleanup(struct nfp_cpp_area *area)
+{
+}
+
+static void priv_area_get(struct nfp_cpp_area *area)
+{
+ struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
+
+ atomic_inc(&priv->refcnt);
+}
+
+static int priv_area_put(struct nfp_cpp_area *area)
+{
+ struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
+
+ if (WARN_ON(!atomic_read(&priv->refcnt)))
+ return 0;
+
+ return atomic_dec_and_test(&priv->refcnt);
+}
+
+static int nfp6000_area_acquire(struct nfp_cpp_area *area)
+{
+ struct nfp6000_pcie *nfp = nfp_cpp_priv(nfp_cpp_area_cpp(area));
+ struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
+ int barnum, err;
+
+ if (priv->bar) {
+ /* Already allocated. */
+ priv_area_get(area);
+ return 0;
+ }
+
+ barnum = nfp_alloc_bar(nfp, priv->target, priv->action, priv->token,
+ priv->offset, priv->size, priv->width.bar, 1);
+
+ if (barnum < 0) {
+ err = barnum;
+ goto err_alloc_bar;
+ }
+ priv->bar = &nfp->bar[barnum];
+
+ /* Calculate offset into BAR. */
+ if (nfp_bar_maptype(priv->bar) ==
+ NFP_PCIE_BAR_PCIE2CPP_MapType_GENERAL) {
+ priv->bar_offset = priv->offset &
+ (NFP_PCIE_P2C_GENERAL_SIZE(priv->bar) - 1);
+ priv->bar_offset += NFP_PCIE_P2C_GENERAL_TARGET_OFFSET(
+ priv->bar, priv->target);
+ priv->bar_offset += NFP_PCIE_P2C_GENERAL_TOKEN_OFFSET(
+ priv->bar, priv->token);
+ } else {
+ priv->bar_offset = priv->offset & priv->bar->mask;
+ }
+
+ /* We don't actually try to acquire the resource area using
+ * request_resource. This would prevent sharing the mapped
+ * BAR between multiple CPP areas and prevent us from
+ * effectively utilizing the limited amount of BAR resources.
+ */
+ priv->phys = nfp_bar_resource_start(priv->bar) + priv->bar_offset;
+ priv->resource.name = nfp_cpp_area_name(area);
+ priv->resource.start = priv->phys;
+ priv->resource.end = priv->resource.start + priv->size - 1;
+ priv->resource.flags = IORESOURCE_MEM;
+
+ /* If the bar is already mapped in, use its mapping */
+ if (priv->bar->iomem)
+ priv->iomem = priv->bar->iomem + priv->bar_offset;
+ else
+ /* Must have been too big. Sub-allocate. */
+ priv->iomem = ioremap_nocache(priv->phys, priv->size);
+
+ if (IS_ERR_OR_NULL(priv->iomem)) {
+ dev_err(nfp->dev, "Can't ioremap() a %d byte region of BAR %d\n",
+ (int)priv->size, priv->bar->index);
+ err = !priv->iomem ? -ENOMEM : PTR_ERR(priv->iomem);
+ priv->iomem = NULL;
+ goto err_iomem_remap;
+ }
+
+ priv_area_get(area);
+ return 0;
+
+err_iomem_remap:
+ nfp_bar_put(nfp, priv->bar);
+ priv->bar = NULL;
+err_alloc_bar:
+ return err;
+}
+
+static void nfp6000_area_release(struct nfp_cpp_area *area)
+{
+ struct nfp6000_pcie *nfp = nfp_cpp_priv(nfp_cpp_area_cpp(area));
+ struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
+
+ if (!priv_area_put(area))
+ return;
+
+ if (!priv->bar->iomem)
+ iounmap(priv->iomem);
+
+ nfp_bar_put(nfp, priv->bar);
+
+ priv->bar = NULL;
+ priv->iomem = NULL;
+}
+
+static phys_addr_t nfp6000_area_phys(struct nfp_cpp_area *area)
+{
+ struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
+
+ return priv->phys;
+}
+
+static void __iomem *nfp6000_area_iomem(struct nfp_cpp_area *area)
+{
+ struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
+
+ return priv->iomem;
+}
+
+static struct resource *nfp6000_area_resource(struct nfp_cpp_area *area)
+{
+ /* Use the BAR resource as the resource for the CPP area.
+ * This enables us to share the BAR among multiple CPP areas
+ * without resource conflicts.
+ */
+ struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
+
+ return priv->bar->resource;
+}
+
+static int nfp6000_area_read(struct nfp_cpp_area *area, void *kernel_vaddr,
+ unsigned long offset, unsigned int length)
+{
+ u64 __maybe_unused *wrptr64 = kernel_vaddr;
+ const u64 __iomem __maybe_unused *rdptr64;
+ struct nfp6000_area_priv *priv;
+ u32 *wrptr32 = kernel_vaddr;
+ const u32 __iomem *rdptr32;
+ int n, width;
+ bool is_64;
+
+ priv = nfp_cpp_area_priv(area);
+ rdptr64 = priv->iomem + offset;
+ rdptr32 = priv->iomem + offset;
+
+ if (offset + length > priv->size)
+ return -EFAULT;
+
+ width = priv->width.read;
+
+ if (width <= 0)
+ return -EINVAL;
+
+ /* Unaligned? Translate to an explicit access */
+ if ((priv->offset + offset) & (width - 1))
+ return nfp_cpp_explicit_read(nfp_cpp_area_cpp(area),
+ NFP_CPP_ID(priv->target,
+ priv->action,
+ priv->token),
+ priv->offset + offset,
+ kernel_vaddr, length, width);
+
+ is_64 = width == TARGET_WIDTH_64;
+
+ /* MU reads via a PCIe2CPP BAR supports 32bit (and other) lengths */
+ if (priv->target == (NFP_CPP_TARGET_ID_MASK & NFP_CPP_TARGET_MU) &&
+ priv->action == NFP_CPP_ACTION_RW)
+ is_64 = false;
+
+ if (is_64) {
+ if (offset % sizeof(u64) != 0 || length % sizeof(u64) != 0)
+ return -EINVAL;
+ } else {
+ if (offset % sizeof(u32) != 0 || length % sizeof(u32) != 0)
+ return -EINVAL;
+ }
+
+ if (WARN_ON(!priv->bar))
+ return -EFAULT;
+
+ if (is_64)
+#ifndef __raw_readq
+ return -EINVAL;
+#else
+ for (n = 0; n < length; n += sizeof(u64))
+ *wrptr64++ = __raw_readq(rdptr64++);
+#endif
+ else
+ for (n = 0; n < length; n += sizeof(u32))
+ *wrptr32++ = __raw_readl(rdptr32++);
+
+ return n;
+}
+
+static int
+nfp6000_area_write(struct nfp_cpp_area *area,
+ const void *kernel_vaddr,
+ unsigned long offset, unsigned int length)
+{
+ const u64 __maybe_unused *rdptr64 = kernel_vaddr;
+ u64 __iomem __maybe_unused *wrptr64;
+ const u32 *rdptr32 = kernel_vaddr;
+ struct nfp6000_area_priv *priv;
+ u32 __iomem *wrptr32;
+ int n, width;
+ bool is_64;
+
+ priv = nfp_cpp_area_priv(area);
+ wrptr64 = priv->iomem + offset;
+ wrptr32 = priv->iomem + offset;
+
+ if (offset + length > priv->size)
+ return -EFAULT;
+
+ width = priv->width.write;
+
+ if (width <= 0)
+ return -EINVAL;
+
+ /* Unaligned? Translate to an explicit access */
+ if ((priv->offset + offset) & (width - 1))
+ return nfp_cpp_explicit_write(nfp_cpp_area_cpp(area),
+ NFP_CPP_ID(priv->target,
+ priv->action,
+ priv->token),
+ priv->offset + offset,
+ kernel_vaddr, length, width);
+
+ is_64 = width == TARGET_WIDTH_64;
+
+ /* MU writes via a PCIe2CPP BAR supports 32bit (and other) lengths */
+ if (priv->target == (NFP_CPP_TARGET_ID_MASK & NFP_CPP_TARGET_MU) &&
+ priv->action == NFP_CPP_ACTION_RW)
+ is_64 = false;
+
+ if (is_64) {
+ if (offset % sizeof(u64) != 0 || length % sizeof(u64) != 0)
+ return -EINVAL;
+ } else {
+ if (offset % sizeof(u32) != 0 || length % sizeof(u32) != 0)
+ return -EINVAL;
+ }
+
+ if (WARN_ON(!priv->bar))
+ return -EFAULT;
+
+ if (is_64)
+#ifndef __raw_writeq
+ return -EINVAL;
+#else
+ for (n = 0; n < length; n += sizeof(u64)) {
+ __raw_writeq(*rdptr64++, wrptr64++);
+ wmb();
+ }
+#endif
+ else
+ for (n = 0; n < length; n += sizeof(u32)) {
+ __raw_writel(*rdptr32++, wrptr32++);
+ wmb();
+ }
+
+ return n;
+}
+
+struct nfp6000_explicit_priv {
+ struct nfp6000_pcie *nfp;
+ struct {
+ int group;
+ int area;
+ } bar;
+ int bitsize;
+ void __iomem *data;
+ void __iomem *addr;
+};
+
+static int nfp6000_explicit_acquire(struct nfp_cpp_explicit *expl)
+{
+ struct nfp6000_pcie *nfp = nfp_cpp_priv(nfp_cpp_explicit_cpp(expl));
+ struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl);
+ int i, j;
+
+ mutex_lock(&nfp->expl.mutex);
+ for (i = 0; i < ARRAY_SIZE(nfp->expl.group); i++) {
+ if (!nfp->expl.group[i].bitsize)
+ continue;
+
+ for (j = 0; j < ARRAY_SIZE(nfp->expl.group[i].free); j++) {
+ u16 data_offset;
+
+ if (!nfp->expl.group[i].free[j])
+ continue;
+
+ priv->nfp = nfp;
+ priv->bar.group = i;
+ priv->bar.area = j;
+ priv->bitsize = nfp->expl.group[i].bitsize - 2;
+
+ data_offset = (priv->bar.group << 9) +
+ (priv->bar.area << 7);
+ priv->data = nfp->expl.data + data_offset;
+ priv->addr = nfp->expl.group[i].addr +
+ (priv->bar.area << priv->bitsize);
+ nfp->expl.group[i].free[j] = false;
+
+ mutex_unlock(&nfp->expl.mutex);
+ return 0;
+ }
+ }
+ mutex_unlock(&nfp->expl.mutex);
+
+ return -EAGAIN;
+}
+
+static void nfp6000_explicit_release(struct nfp_cpp_explicit *expl)
+{
+ struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl);
+ struct nfp6000_pcie *nfp = priv->nfp;
+
+ mutex_lock(&nfp->expl.mutex);
+ nfp->expl.group[priv->bar.group].free[priv->bar.area] = true;
+ mutex_unlock(&nfp->expl.mutex);
+}
+
+static int nfp6000_explicit_put(struct nfp_cpp_explicit *expl,
+ const void *buff, size_t len)
+{
+ struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl);
+ const u32 *src = buff;
+ size_t i;
+
+ for (i = 0; i < len; i += sizeof(u32))
+ writel(*(src++), priv->data + i);
+
+ return i;
+}
+
+static int
+nfp6000_explicit_do(struct nfp_cpp_explicit *expl,
+ const struct nfp_cpp_explicit_command *cmd, u64 address)
+{
+ struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl);
+ u8 signal_master, signal_ref, data_master;
+ struct nfp6000_pcie *nfp = priv->nfp;
+ int sigmask = 0;
+ u16 data_ref;
+ u32 csr[3];
+
+ if (cmd->siga_mode)
+ sigmask |= 1 << cmd->siga;
+ if (cmd->sigb_mode)
+ sigmask |= 1 << cmd->sigb;
+
+ signal_master = cmd->signal_master;
+ if (!signal_master)
+ signal_master = nfp->expl.master_id;
+
+ signal_ref = cmd->signal_ref;
+ if (signal_master == nfp->expl.master_id)
+ signal_ref = nfp->expl.signal_ref +
+ ((priv->bar.group * 4 + priv->bar.area) << 1);
+
+ data_master = cmd->data_master;
+ if (!data_master)
+ data_master = nfp->expl.master_id;
+
+ data_ref = cmd->data_ref;
+ if (data_master == nfp->expl.master_id)
+ data_ref = 0x1000 +
+ (priv->bar.group << 9) + (priv->bar.area << 7);
+
+ csr[0] = NFP_PCIE_BAR_EXPLICIT_BAR0_SignalType(sigmask) |
+ NFP_PCIE_BAR_EXPLICIT_BAR0_Token(
+ NFP_CPP_ID_TOKEN_of(cmd->cpp_id)) |
+ NFP_PCIE_BAR_EXPLICIT_BAR0_Address(address >> 16);
+
+ csr[1] = NFP_PCIE_BAR_EXPLICIT_BAR1_SignalRef(signal_ref) |
+ NFP_PCIE_BAR_EXPLICIT_BAR1_DataMaster(data_master) |
+ NFP_PCIE_BAR_EXPLICIT_BAR1_DataRef(data_ref);
+
+ csr[2] = NFP_PCIE_BAR_EXPLICIT_BAR2_Target(
+ NFP_CPP_ID_TARGET_of(cmd->cpp_id)) |
+ NFP_PCIE_BAR_EXPLICIT_BAR2_Action(
+ NFP_CPP_ID_ACTION_of(cmd->cpp_id)) |
+ NFP_PCIE_BAR_EXPLICIT_BAR2_Length(cmd->len) |
+ NFP_PCIE_BAR_EXPLICIT_BAR2_ByteMask(cmd->byte_mask) |
+ NFP_PCIE_BAR_EXPLICIT_BAR2_SignalMaster(signal_master);
+
+ if (nfp->iomem.csr) {
+ writel(csr[0], nfp->iomem.csr +
+ NFP_PCIE_BAR_EXPLICIT_BAR0(priv->bar.group,
+ priv->bar.area));
+ writel(csr[1], nfp->iomem.csr +
+ NFP_PCIE_BAR_EXPLICIT_BAR1(priv->bar.group,
+ priv->bar.area));
+ writel(csr[2], nfp->iomem.csr +
+ NFP_PCIE_BAR_EXPLICIT_BAR2(priv->bar.group,
+ priv->bar.area));
+ /* Readback to ensure BAR is flushed */
+ readl(nfp->iomem.csr +
+ NFP_PCIE_BAR_EXPLICIT_BAR0(priv->bar.group,
+ priv->bar.area));
+ readl(nfp->iomem.csr +
+ NFP_PCIE_BAR_EXPLICIT_BAR1(priv->bar.group,
+ priv->bar.area));
+ readl(nfp->iomem.csr +
+ NFP_PCIE_BAR_EXPLICIT_BAR2(priv->bar.group,
+ priv->bar.area));
+ } else {
+ pci_write_config_dword(nfp->pdev, 0x400 +
+ NFP_PCIE_BAR_EXPLICIT_BAR0(
+ priv->bar.group, priv->bar.area),
+ csr[0]);
+
+ pci_write_config_dword(nfp->pdev, 0x400 +
+ NFP_PCIE_BAR_EXPLICIT_BAR1(
+ priv->bar.group, priv->bar.area),
+ csr[1]);
+
+ pci_write_config_dword(nfp->pdev, 0x400 +
+ NFP_PCIE_BAR_EXPLICIT_BAR2(
+ priv->bar.group, priv->bar.area),
+ csr[2]);
+ }
+
+ /* Issue the 'kickoff' transaction */
+ readb(priv->addr + (address & ((1 << priv->bitsize) - 1)));
+
+ return sigmask;
+}
+
+static int nfp6000_explicit_get(struct nfp_cpp_explicit *expl,
+ void *buff, size_t len)
+{
+ struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl);
+ u32 *dst = buff;
+ size_t i;
+
+ for (i = 0; i < len; i += sizeof(u32))
+ *(dst++) = readl(priv->data + i);
+
+ return i;
+}
+
+static int nfp6000_init(struct nfp_cpp *cpp)
+{
+ nfp_cpp_area_cache_add(cpp, SZ_64K);
+ nfp_cpp_area_cache_add(cpp, SZ_64K);
+ nfp_cpp_area_cache_add(cpp, SZ_256K);
+
+ return 0;
+}
+
+static void nfp6000_free(struct nfp_cpp *cpp)
+{
+ struct nfp6000_pcie *nfp = nfp_cpp_priv(cpp);
+
+ disable_bars(nfp);
+ kfree(nfp);
+}
+
+static void nfp6000_read_serial(struct device *dev, u8 *serial)
+{
+ struct pci_dev *pdev = to_pci_dev(dev);
+ int pos;
+ u32 reg;
+
+ pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_DSN);
+ if (!pos) {
+ memset(serial, 0, NFP_SERIAL_LEN);
+ return;
+ }
+
+ pci_read_config_dword(pdev, pos + 4, ®);
+ put_unaligned_be16(reg >> 16, serial + 4);
+ pci_read_config_dword(pdev, pos + 8, ®);
+ put_unaligned_be32(reg, serial);
+}
+
+static u16 nfp6000_get_interface(struct device *dev)
+{
+ struct pci_dev *pdev = to_pci_dev(dev);
+ int pos;
+ u32 reg;
+
+ pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_DSN);
+ if (!pos)
+ return NFP_CPP_INTERFACE(NFP_CPP_INTERFACE_TYPE_PCI, 0, 0xff);
+
+ pci_read_config_dword(pdev, pos + 4, ®);
+
+ return reg & 0xffff;
+}
+
+static const struct nfp_cpp_operations nfp6000_pcie_ops = {
+ .owner = THIS_MODULE,
+
+ .init = nfp6000_init,
+ .free = nfp6000_free,
+
+ .read_serial = nfp6000_read_serial,
+ .get_interface = nfp6000_get_interface,
+
+ .area_priv_size = sizeof(struct nfp6000_area_priv),
+ .area_init = nfp6000_area_init,
+ .area_cleanup = nfp6000_area_cleanup,
+ .area_acquire = nfp6000_area_acquire,
+ .area_release = nfp6000_area_release,
+ .area_phys = nfp6000_area_phys,
+ .area_iomem = nfp6000_area_iomem,
+ .area_resource = nfp6000_area_resource,
+ .area_read = nfp6000_area_read,
+ .area_write = nfp6000_area_write,
+
+ .explicit_priv_size = sizeof(struct nfp6000_explicit_priv),
+ .explicit_acquire = nfp6000_explicit_acquire,
+ .explicit_release = nfp6000_explicit_release,
+ .explicit_put = nfp6000_explicit_put,
+ .explicit_do = nfp6000_explicit_do,
+ .explicit_get = nfp6000_explicit_get,
+};
+
+/**
+ * nfp_cpp_from_nfp6000_pcie() - Build a NFP CPP bus from a NFP6000 PCI device
+ * @pdev: NFP6000 PCI device
+ *
+ * Return: NFP CPP handle
+ */
+struct nfp_cpp *nfp_cpp_from_nfp6000_pcie(struct pci_dev *pdev)
+{
+ struct nfp6000_pcie *nfp;
+ u16 interface;
+ int err;
+
+ /* Finished with card initialization. */
+ dev_info(&pdev->dev,
+ "Netronome Flow Processor NFP4000/NFP6000 PCIe Card Probe\n");
+
+ nfp = kzalloc(sizeof(*nfp), GFP_KERNEL);
+ if (!nfp) {
+ err = -ENOMEM;
+ goto err_ret;
+ }
+
+ nfp->dev = &pdev->dev;
+ nfp->pdev = pdev;
+ init_waitqueue_head(&nfp->bar_waiters);
+ spin_lock_init(&nfp->bar_lock);
+
+ interface = nfp6000_get_interface(&pdev->dev);
+
+ if (NFP_CPP_INTERFACE_TYPE_of(interface) !=
+ NFP_CPP_INTERFACE_TYPE_PCI) {
+ dev_err(&pdev->dev,
+ "Interface type %d is not the expected %d\n",
+ NFP_CPP_INTERFACE_TYPE_of(interface),
+ NFP_CPP_INTERFACE_TYPE_PCI);
+ err = -ENODEV;
+ goto err_free_nfp;
+ }
+
+ if (NFP_CPP_INTERFACE_CHANNEL_of(interface) !=
+ NFP_CPP_INTERFACE_CHANNEL_PEROPENER) {
+ dev_err(&pdev->dev, "Interface channel %d is not the expected %d\n",
+ NFP_CPP_INTERFACE_CHANNEL_of(interface),
+ NFP_CPP_INTERFACE_CHANNEL_PEROPENER);
+ err = -ENODEV;
+ goto err_free_nfp;
+ }
+
+ err = enable_bars(nfp, interface);
+ if (err)
+ goto err_free_nfp;
+
+ /* Probe for all the common NFP devices */
+ return nfp_cpp_from_operations(&nfp6000_pcie_ops, &pdev->dev, nfp);
+
+err_free_nfp:
+ kfree(nfp);
+err_ret:
+ dev_err(&pdev->dev, "NFP6000 PCI setup failed\n");
+ return ERR_PTR(err);
+}
--- /dev/null
+/*
+ * Copyright (C) 2015-2017 Netronome Systems, Inc.
+ *
+ * This software is dual licensed under the GNU General License Version 2,
+ * June 1991 as shown in the file COPYING in the top-level directory of this
+ * source tree or the BSD 2-Clause License provided below. You have the
+ * option to license this software under the complete terms of either license.
+ *
+ * The BSD 2-Clause License:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+/*
+ * nfp6000_pcie.h
+ * Author: Jason McMullan <jason.mcmullan@netronome.com>
+ */
+
+#ifndef NFP6000_PCIE_H
+#define NFP6000_PCIE_H
+
+#include "nfp_cpp.h"
+
+struct nfp_cpp *nfp_cpp_from_nfp6000_pcie(struct pci_dev *pdev);
+
+#endif /* NFP6000_PCIE_H */
--- /dev/null
+/*
+ * Copyright (C) 2015-2017 Netronome Systems, Inc.
+ *
+ * This software is dual licensed under the GNU General License Version 2,
+ * June 1991 as shown in the file COPYING in the top-level directory of this
+ * source tree or the BSD 2-Clause License provided below. You have the
+ * option to license this software under the complete terms of either license.
+ *
+ * The BSD 2-Clause License:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+/*
+ * nfp_arm.h
+ * Definitions for ARM-based registers and memory spaces
+ */
+
+#ifndef NFP_ARM_H
+#define NFP_ARM_H
+
+#define NFP_ARM_QUEUE(_q) (0x100000 + (0x800 * ((_q) & 0xff)))
+#define NFP_ARM_IM 0x200000
+#define NFP_ARM_EM 0x300000
+#define NFP_ARM_GCSR 0x400000
+#define NFP_ARM_MPCORE 0x800000
+#define NFP_ARM_PL310 0xa00000
+/* Register Type: BulkBARConfig */
+#define NFP_ARM_GCSR_BULK_BAR(_bar) (0x0 + (0x4 * ((_bar) & 0x7)))
+#define NFP_ARM_GCSR_BULK_BAR_TYPE (0x1 << 31)
+#define NFP_ARM_GCSR_BULK_BAR_TYPE_BULK (0x0)
+#define NFP_ARM_GCSR_BULK_BAR_TYPE_EXPA (0x80000000)
+#define NFP_ARM_GCSR_BULK_BAR_TGT(_x) (((_x) & 0xf) << 27)
+#define NFP_ARM_GCSR_BULK_BAR_TGT_of(_x) (((_x) >> 27) & 0xf)
+#define NFP_ARM_GCSR_BULK_BAR_TOK(_x) (((_x) & 0x3) << 25)
+#define NFP_ARM_GCSR_BULK_BAR_TOK_of(_x) (((_x) >> 25) & 0x3)
+#define NFP_ARM_GCSR_BULK_BAR_LEN (0x1 << 24)
+#define NFP_ARM_GCSR_BULK_BAR_LEN_32BIT (0x0)
+#define NFP_ARM_GCSR_BULK_BAR_LEN_64BIT (0x1000000)
+#define NFP_ARM_GCSR_BULK_BAR_ADDR(_x) ((_x) & 0x7ff)
+#define NFP_ARM_GCSR_BULK_BAR_ADDR_of(_x) ((_x) & 0x7ff)
+/* Register Type: ExpansionBARConfig */
+#define NFP_ARM_GCSR_EXPA_BAR(_bar) (0x20 + (0x4 * ((_bar) & 0xf)))
+#define NFP_ARM_GCSR_EXPA_BAR_TYPE (0x1 << 31)
+#define NFP_ARM_GCSR_EXPA_BAR_TYPE_EXPA (0x0)
+#define NFP_ARM_GCSR_EXPA_BAR_TYPE_EXPL (0x80000000)
+#define NFP_ARM_GCSR_EXPA_BAR_TGT(_x) (((_x) & 0xf) << 27)
+#define NFP_ARM_GCSR_EXPA_BAR_TGT_of(_x) (((_x) >> 27) & 0xf)
+#define NFP_ARM_GCSR_EXPA_BAR_TOK(_x) (((_x) & 0x3) << 25)
+#define NFP_ARM_GCSR_EXPA_BAR_TOK_of(_x) (((_x) >> 25) & 0x3)
+#define NFP_ARM_GCSR_EXPA_BAR_LEN (0x1 << 24)
+#define NFP_ARM_GCSR_EXPA_BAR_LEN_32BIT (0x0)
+#define NFP_ARM_GCSR_EXPA_BAR_LEN_64BIT (0x1000000)
+#define NFP_ARM_GCSR_EXPA_BAR_ACT(_x) (((_x) & 0x1f) << 19)
+#define NFP_ARM_GCSR_EXPA_BAR_ACT_of(_x) (((_x) >> 19) & 0x1f)
+#define NFP_ARM_GCSR_EXPA_BAR_ACT_DERIVED (0)
+#define NFP_ARM_GCSR_EXPA_BAR_ADDR(_x) ((_x) & 0x7fff)
+#define NFP_ARM_GCSR_EXPA_BAR_ADDR_of(_x) ((_x) & 0x7fff)
+/* Register Type: ExplicitBARConfig0_Reg */
+#define NFP_ARM_GCSR_EXPL0_BAR(_bar) (0x60 + (0x4 * ((_bar) & 0x7)))
+#define NFP_ARM_GCSR_EXPL0_BAR_ADDR(_x) ((_x) & 0x3ffff)
+#define NFP_ARM_GCSR_EXPL0_BAR_ADDR_of(_x) ((_x) & 0x3ffff)
+/* Register Type: ExplicitBARConfig1_Reg */
+#define NFP_ARM_GCSR_EXPL1_BAR(_bar) (0x80 + (0x4 * ((_bar) & 0x7)))
+#define NFP_ARM_GCSR_EXPL1_BAR_POSTED (0x1 << 31)
+#define NFP_ARM_GCSR_EXPL1_BAR_SIGNAL_REF(_x) (((_x) & 0x7f) << 24)
+#define NFP_ARM_GCSR_EXPL1_BAR_SIGNAL_REF_of(_x) (((_x) >> 24) & 0x7f)
+#define NFP_ARM_GCSR_EXPL1_BAR_DATA_MASTER(_x) (((_x) & 0xff) << 16)
+#define NFP_ARM_GCSR_EXPL1_BAR_DATA_MASTER_of(_x) (((_x) >> 16) & 0xff)
+#define NFP_ARM_GCSR_EXPL1_BAR_DATA_REF(_x) ((_x) & 0x3fff)
+#define NFP_ARM_GCSR_EXPL1_BAR_DATA_REF_of(_x) ((_x) & 0x3fff)
+/* Register Type: ExplicitBARConfig2_Reg */
+#define NFP_ARM_GCSR_EXPL2_BAR(_bar) (0xa0 + (0x4 * ((_bar) & 0x7)))
+#define NFP_ARM_GCSR_EXPL2_BAR_TGT(_x) (((_x) & 0xf) << 28)
+#define NFP_ARM_GCSR_EXPL2_BAR_TGT_of(_x) (((_x) >> 28) & 0xf)
+#define NFP_ARM_GCSR_EXPL2_BAR_ACT(_x) (((_x) & 0x1f) << 23)
+#define NFP_ARM_GCSR_EXPL2_BAR_ACT_of(_x) (((_x) >> 23) & 0x1f)
+#define NFP_ARM_GCSR_EXPL2_BAR_LEN(_x) (((_x) & 0x1f) << 18)
+#define NFP_ARM_GCSR_EXPL2_BAR_LEN_of(_x) (((_x) >> 18) & 0x1f)
+#define NFP_ARM_GCSR_EXPL2_BAR_BYTE_MASK(_x) (((_x) & 0xff) << 10)
+#define NFP_ARM_GCSR_EXPL2_BAR_BYTE_MASK_of(_x) (((_x) >> 10) & 0xff)
+#define NFP_ARM_GCSR_EXPL2_BAR_TOK(_x) (((_x) & 0x3) << 8)
+#define NFP_ARM_GCSR_EXPL2_BAR_TOK_of(_x) (((_x) >> 8) & 0x3)
+#define NFP_ARM_GCSR_EXPL2_BAR_SIGNAL_MASTER(_x) ((_x) & 0xff)
+#define NFP_ARM_GCSR_EXPL2_BAR_SIGNAL_MASTER_of(_x) ((_x) & 0xff)
+/* Register Type: PostedCommandSignal */
+#define NFP_ARM_GCSR_EXPL_POST(_bar) (0xc0 + (0x4 * ((_bar) & 0x7)))
+#define NFP_ARM_GCSR_EXPL_POST_SIG_B(_x) (((_x) & 0x7f) << 25)
+#define NFP_ARM_GCSR_EXPL_POST_SIG_B_of(_x) (((_x) >> 25) & 0x7f)
+#define NFP_ARM_GCSR_EXPL_POST_SIG_B_BUS (0x1 << 24)
+#define NFP_ARM_GCSR_EXPL_POST_SIG_B_BUS_PULL (0x0)
+#define NFP_ARM_GCSR_EXPL_POST_SIG_B_BUS_PUSH (0x1000000)
+#define NFP_ARM_GCSR_EXPL_POST_SIG_A(_x) (((_x) & 0x7f) << 17)
+#define NFP_ARM_GCSR_EXPL_POST_SIG_A_of(_x) (((_x) >> 17) & 0x7f)
+#define NFP_ARM_GCSR_EXPL_POST_SIG_A_BUS (0x1 << 16)
+#define NFP_ARM_GCSR_EXPL_POST_SIG_A_BUS_PULL (0x0)
+#define NFP_ARM_GCSR_EXPL_POST_SIG_A_BUS_PUSH (0x10000)
+#define NFP_ARM_GCSR_EXPL_POST_SIG_B_RCVD (0x1 << 7)
+#define NFP_ARM_GCSR_EXPL_POST_SIG_B_VALID (0x1 << 6)
+#define NFP_ARM_GCSR_EXPL_POST_SIG_A_RCVD (0x1 << 5)
+#define NFP_ARM_GCSR_EXPL_POST_SIG_A_VALID (0x1 << 4)
+#define NFP_ARM_GCSR_EXPL_POST_CMD_COMPLETE (0x1)
+/* Register Type: MPCoreBaseAddress */
+#define NFP_ARM_GCSR_MPCORE_BASE 0x00e0
+#define NFP_ARM_GCSR_MPCORE_BASE_ADDR(_x) (((_x) & 0x7ffff) << 13)
+#define NFP_ARM_GCSR_MPCORE_BASE_ADDR_of(_x) (((_x) >> 13) & 0x7ffff)
+/* Register Type: PL310BaseAddress */
+#define NFP_ARM_GCSR_PL310_BASE 0x00e4
+#define NFP_ARM_GCSR_PL310_BASE_ADDR(_x) (((_x) & 0xfffff) << 12)
+#define NFP_ARM_GCSR_PL310_BASE_ADDR_of(_x) (((_x) >> 12) & 0xfffff)
+/* Register Type: MPCoreConfig */
+#define NFP_ARM_GCSR_MP0_CFG 0x00e8
+#define NFP_ARM_GCSR_MP0_CFG_SPI_BOOT (0x1 << 14)
+#define NFP_ARM_GCSR_MP0_CFG_ENDIAN(_x) (((_x) & 0x3) << 12)
+#define NFP_ARM_GCSR_MP0_CFG_ENDIAN_of(_x) (((_x) >> 12) & 0x3)
+#define NFP_ARM_GCSR_MP0_CFG_ENDIAN_LITTLE (0)
+#define NFP_ARM_GCSR_MP0_CFG_ENDIAN_BIG (1)
+#define NFP_ARM_GCSR_MP0_CFG_RESET_VECTOR (0x1 << 8)
+#define NFP_ARM_GCSR_MP0_CFG_RESET_VECTOR_LO (0x0)
+#define NFP_ARM_GCSR_MP0_CFG_RESET_VECTOR_HI (0x100)
+#define NFP_ARM_GCSR_MP0_CFG_OUTCLK_EN(_x) (((_x) & 0xf) << 4)
+#define NFP_ARM_GCSR_MP0_CFG_OUTCLK_EN_of(_x) (((_x) >> 4) & 0xf)
+#define NFP_ARM_GCSR_MP0_CFG_ARMID(_x) ((_x) & 0xf)
+#define NFP_ARM_GCSR_MP0_CFG_ARMID_of(_x) ((_x) & 0xf)
+/* Register Type: MPCoreIDCacheDataError */
+#define NFP_ARM_GCSR_MP0_CACHE_ERR 0x00ec
+#define NFP_ARM_GCSR_MP0_CACHE_ERR_MP0_D7 (0x1 << 15)
+#define NFP_ARM_GCSR_MP0_CACHE_ERR_MP0_D6 (0x1 << 14)
+#define NFP_ARM_GCSR_MP0_CACHE_ERR_MP0_D5 (0x1 << 13)
+#define NFP_ARM_GCSR_MP0_CACHE_ERR_MP0_D4 (0x1 << 12)
+#define NFP_ARM_GCSR_MP0_CACHE_ERR_MP0_D3 (0x1 << 11)
+#define NFP_ARM_GCSR_MP0_CACHE_ERR_MP0_D2 (0x1 << 10)
+#define NFP_ARM_GCSR_MP0_CACHE_ERR_MP0_D1 (0x1 << 9)
+#define NFP_ARM_GCSR_MP0_CACHE_ERR_MP0_D0 (0x1 << 8)
+#define NFP_ARM_GCSR_MP0_CACHE_ERR_MP0_I7 (0x1 << 7)
+#define NFP_ARM_GCSR_MP0_CACHE_ERR_MP0_I6 (0x1 << 6)
+#define NFP_ARM_GCSR_MP0_CACHE_ERR_MP0_I5 (0x1 << 5)
+#define NFP_ARM_GCSR_MP0_CACHE_ERR_MP0_I4 (0x1 << 4)
+#define NFP_ARM_GCSR_MP0_CACHE_ERR_MP0_I3 (0x1 << 3)
+#define NFP_ARM_GCSR_MP0_CACHE_ERR_MP0_I2 (0x1 << 2)
+#define NFP_ARM_GCSR_MP0_CACHE_ERR_MP0_I1 (0x1 << 1)
+#define NFP_ARM_GCSR_MP0_CACHE_ERR_MP0_I0 (0x1)
+/* Register Type: ARMDFT */
+#define NFP_ARM_GCSR_DFT 0x0100
+#define NFP_ARM_GCSR_DFT_DBG_REQ (0x1 << 20)
+#define NFP_ARM_GCSR_DFT_DBG_EN (0x1 << 19)
+#define NFP_ARM_GCSR_DFT_WFE_EVT_TRG (0x1 << 18)
+#define NFP_ARM_GCSR_DFT_ETM_WFI_RDY (0x1 << 17)
+#define NFP_ARM_GCSR_DFT_ETM_PWR_ON (0x1 << 16)
+#define NFP_ARM_GCSR_DFT_BIST_FAIL_of(_x) (((_x) >> 8) & 0xf)
+#define NFP_ARM_GCSR_DFT_BIST_DONE_of(_x) (((_x) >> 4) & 0xf)
+#define NFP_ARM_GCSR_DFT_BIST_RUN(_x) ((_x) & 0x7)
+#define NFP_ARM_GCSR_DFT_BIST_RUN_of(_x) ((_x) & 0x7)
+
+/* Gasket CSRs */
+/* NOTE: These cannot be remapped, and are always at this location.
+ */
+#define NFP_ARM_GCSR_START (0xd6000000 + NFP_ARM_GCSR)
+#define NFP_ARM_GCSR_SIZE SZ_64K
+
+/* BAR CSRs
+ */
+#define NFP_ARM_GCSR_BULK_BITS 11
+#define NFP_ARM_GCSR_EXPA_BITS 15
+#define NFP_ARM_GCSR_EXPL_BITS 18
+
+#define NFP_ARM_GCSR_BULK_SHIFT (40 - 11)
+#define NFP_ARM_GCSR_EXPA_SHIFT (40 - 15)
+#define NFP_ARM_GCSR_EXPL_SHIFT (40 - 18)
+
+#define NFP_ARM_GCSR_BULK_SIZE (1 << NFP_ARM_GCSR_BULK_SHIFT)
+#define NFP_ARM_GCSR_EXPA_SIZE (1 << NFP_ARM_GCSR_EXPA_SHIFT)
+#define NFP_ARM_GCSR_EXPL_SIZE (1 << NFP_ARM_GCSR_EXPL_SHIFT)
+
+#define NFP_ARM_GCSR_EXPL2_CSR(target, action, length, \
+ byte_mask, token, signal_master) \
+ (NFP_ARM_GCSR_EXPL2_BAR_TGT(target) | \
+ NFP_ARM_GCSR_EXPL2_BAR_ACT(action) | \
+ NFP_ARM_GCSR_EXPL2_BAR_LEN(length) | \
+ NFP_ARM_GCSR_EXPL2_BAR_BYTE_MASK(byte_mask) | \
+ NFP_ARM_GCSR_EXPL2_BAR_TOK(token) | \
+ NFP_ARM_GCSR_EXPL2_BAR_SIGNAL_MASTER(signal_master))
+#define NFP_ARM_GCSR_EXPL1_CSR(posted, signal_ref, data_master, data_ref) \
+ (((posted) ? NFP_ARM_GCSR_EXPL1_BAR_POSTED : 0) | \
+ NFP_ARM_GCSR_EXPL1_BAR_SIGNAL_REF(signal_ref) | \
+ NFP_ARM_GCSR_EXPL1_BAR_DATA_MASTER(data_master) | \
+ NFP_ARM_GCSR_EXPL1_BAR_DATA_REF(data_ref))
+#define NFP_ARM_GCSR_EXPL0_CSR(address) \
+ NFP_ARM_GCSR_EXPL0_BAR_ADDR((address) >> NFP_ARM_GCSR_EXPL_SHIFT)
+#define NFP_ARM_GCSR_EXPL_POST_EXPECT_A(sig_ref, is_push, is_required) \
+ (NFP_ARM_GCSR_EXPL_POST_SIG_A(sig_ref) | \
+ ((is_push) ? NFP_ARM_GCSR_EXPL_POST_SIG_A_BUS_PUSH : \
+ NFP_ARM_GCSR_EXPL_POST_SIG_A_BUS_PULL) | \
+ ((is_required) ? NFP_ARM_GCSR_EXPL_POST_SIG_A_VALID : 0))
+#define NFP_ARM_GCSR_EXPL_POST_EXPECT_B(sig_ref, is_push, is_required) \
+ (NFP_ARM_GCSR_EXPL_POST_SIG_B(sig_ref) | \
+ ((is_push) ? NFP_ARM_GCSR_EXPL_POST_SIG_B_BUS_PUSH : \
+ NFP_ARM_GCSR_EXPL_POST_SIG_B_BUS_PULL) | \
+ ((is_required) ? NFP_ARM_GCSR_EXPL_POST_SIG_B_VALID : 0))
+
+#define NFP_ARM_GCSR_EXPA_CSR(mode, target, token, is_64, action, address) \
+ (((mode) ? NFP_ARM_GCSR_EXPA_BAR_TYPE_EXPL : \
+ NFP_ARM_GCSR_EXPA_BAR_TYPE_EXPA) | \
+ NFP_ARM_GCSR_EXPA_BAR_TGT(target) | \
+ NFP_ARM_GCSR_EXPA_BAR_TOK(token) | \
+ ((is_64) ? NFP_ARM_GCSR_EXPA_BAR_LEN_64BIT : \
+ NFP_ARM_GCSR_EXPA_BAR_LEN_32BIT) | \
+ NFP_ARM_GCSR_EXPA_BAR_ACT(action) | \
+ NFP_ARM_GCSR_EXPA_BAR_ADDR((address) >> NFP_ARM_GCSR_EXPA_SHIFT))
+
+#define NFP_ARM_GCSR_BULK_CSR(mode, target, token, is_64, address) \
+ (((mode) ? NFP_ARM_GCSR_BULK_BAR_TYPE_EXPA : \
+ NFP_ARM_GCSR_BULK_BAR_TYPE_BULK) | \
+ NFP_ARM_GCSR_BULK_BAR_TGT(target) | \
+ NFP_ARM_GCSR_BULK_BAR_TOK(token) | \
+ ((is_64) ? NFP_ARM_GCSR_BULK_BAR_LEN_64BIT : \
+ NFP_ARM_GCSR_BULK_BAR_LEN_32BIT) | \
+ NFP_ARM_GCSR_BULK_BAR_ADDR((address) >> NFP_ARM_GCSR_BULK_SHIFT))
+
+ /* MP Core CSRs */
+#define NFP_ARM_MPCORE_SIZE SZ_128K
+
+ /* PL320 CSRs */
+#define NFP_ARM_PCSR_SIZE SZ_64K
+
+#endif /* NFP_ARM_H */
--- /dev/null
+/*
+ * Copyright (C) 2015-2017 Netronome Systems, Inc.
+ *
+ * This software is dual licensed under the GNU General License Version 2,
+ * June 1991 as shown in the file COPYING in the top-level directory of this
+ * source tree or the BSD 2-Clause License provided below. You have the
+ * option to license this software under the complete terms of either license.
+ *
+ * The BSD 2-Clause License:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+/*
+ * nfp_cpp.h
+ * Interface for low-level NFP CPP access.
+ * Authors: Jason McMullan <jason.mcmullan@netronome.com>
+ * Rolf Neugebauer <rolf.neugebauer@netronome.com>
+ */
+#ifndef __NFP_CPP_H__
+#define __NFP_CPP_H__
+
+#include <linux/ctype.h>
+#include <linux/types.h>
+
+#ifndef NFP_SUBSYS
+#define NFP_SUBSYS "nfp"
+#endif
+
+#define nfp_err(cpp, fmt, args...) \
+ dev_err(nfp_cpp_device(cpp)->parent, NFP_SUBSYS ": " fmt, ## args)
+#define nfp_warn(cpp, fmt, args...) \
+ dev_warn(nfp_cpp_device(cpp)->parent, NFP_SUBSYS ": " fmt, ## args)
+#define nfp_info(cpp, fmt, args...) \
+ dev_info(nfp_cpp_device(cpp)->parent, NFP_SUBSYS ": " fmt, ## args)
+#define nfp_dbg(cpp, fmt, args...) \
+ dev_dbg(nfp_cpp_device(cpp)->parent, NFP_SUBSYS ": " fmt, ## args)
+
+#define PCI_64BIT_BAR_COUNT 3
+
+/* NFP hardware vendor/device ids.
+ */
+#define PCI_DEVICE_NFP4000 0x4000
+#define PCI_DEVICE_NFP6000 0x6000
+
+#define NFP_CPP_NUM_TARGETS 16
+
+struct device;
+
+struct nfp_cpp_area;
+struct nfp_cpp;
+struct resource;
+
+/* Wildcard indicating a CPP read or write action
+ *
+ * The action used will be either read or write depending on whether a
+ * read or write instruction/call is performed on the NFP_CPP_ID. It
+ * is recomended that the RW action is used even if all actions to be
+ * performed on a NFP_CPP_ID are known to be only reads or writes.
+ * Doing so will in many cases save NFP CPP internal software
+ * resources.
+ */
+#define NFP_CPP_ACTION_RW 32
+
+#define NFP_CPP_TARGET_ID_MASK 0x1f
+
+/**
+ * NFP_CPP_ID() - pack target, token, and action into a CPP ID.
+ * @target: NFP CPP target id
+ * @action: NFP CPP action id
+ * @token: NFP CPP token id
+ *
+ * Create a 32-bit CPP identifier representing the access to be made.
+ * These identifiers are used as parameters to other NFP CPP
+ * functions. Some CPP devices may allow wildcard identifiers to be
+ * specified.
+ *
+ * Return: NFP CPP ID
+ */
+#define NFP_CPP_ID(target, action, token) \
+ ((((target) & 0x7f) << 24) | (((token) & 0xff) << 16) | \
+ (((action) & 0xff) << 8))
+
+/**
+ * NFP_CPP_ISLAND_ID() - pack target, token, action, and island into a CPP ID.
+ * @target: NFP CPP target id
+ * @action: NFP CPP action id
+ * @token: NFP CPP token id
+ * @island: NFP CPP island id
+ *
+ * Create a 32-bit CPP identifier representing the access to be made.
+ * These identifiers are used as parameters to other NFP CPP
+ * functions. Some CPP devices may allow wildcard identifiers to be
+ * specified.
+ *
+ * Return: NFP CPP ID
+ */
+#define NFP_CPP_ISLAND_ID(target, action, token, island) \
+ ((((target) & 0x7f) << 24) | (((token) & 0xff) << 16) | \
+ (((action) & 0xff) << 8) | (((island) & 0xff) << 0))
+
+/**
+ * NFP_CPP_ID_TARGET_of() - Return the NFP CPP target of a NFP CPP ID
+ * @id: NFP CPP ID
+ *
+ * Return: NFP CPP target
+ */
+static inline u8 NFP_CPP_ID_TARGET_of(u32 id)
+{
+ return (id >> 24) & NFP_CPP_TARGET_ID_MASK;
+}
+
+/**
+ * NFP_CPP_ID_TOKEN_of() - Return the NFP CPP token of a NFP CPP ID
+ * @id: NFP CPP ID
+ * Return: NFP CPP token
+ */
+static inline u8 NFP_CPP_ID_TOKEN_of(u32 id)
+{
+ return (id >> 16) & 0xff;
+}
+
+/**
+ * NFP_CPP_ID_ACTION_of() - Return the NFP CPP action of a NFP CPP ID
+ * @id: NFP CPP ID
+ *
+ * Return: NFP CPP action
+ */
+static inline u8 NFP_CPP_ID_ACTION_of(u32 id)
+{
+ return (id >> 8) & 0xff;
+}
+
+/**
+ * NFP_CPP_ID_ISLAND_of() - Return the NFP CPP island of a NFP CPP ID
+ * @id: NFP CPP ID
+ *
+ * Return: NFP CPP island
+ */
+static inline u8 NFP_CPP_ID_ISLAND_of(u32 id)
+{
+ return (id >> 0) & 0xff;
+}
+
+/* NFP Interface types - logical interface for this CPP connection
+ * 4 bits are reserved for interface type.
+ */
+#define NFP_CPP_INTERFACE_TYPE_INVALID 0x0
+#define NFP_CPP_INTERFACE_TYPE_PCI 0x1
+#define NFP_CPP_INTERFACE_TYPE_ARM 0x2
+#define NFP_CPP_INTERFACE_TYPE_RPC 0x3
+#define NFP_CPP_INTERFACE_TYPE_ILA 0x4
+
+/**
+ * NFP_CPP_INTERFACE() - Construct a 16-bit NFP Interface ID
+ * @type: NFP Interface Type
+ * @unit: Unit identifier for the interface type
+ * @channel: Channel identifier for the interface unit
+ *
+ * Interface IDs consists of 4 bits of interface type,
+ * 4 bits of unit identifier, and 8 bits of channel identifier.
+ *
+ * The NFP Interface ID is used in the implementation of
+ * NFP CPP API mutexes, which use the MU Atomic CompareAndWrite
+ * operation - hence the limit to 16 bits to be able to
+ * use the NFP Interface ID as a lock owner.
+ *
+ * Return: Interface ID
+ */
+#define NFP_CPP_INTERFACE(type, unit, channel) \
+ ((((type) & 0xf) << 12) | \
+ (((unit) & 0xf) << 8) | \
+ (((channel) & 0xff) << 0))
+
+/**
+ * NFP_CPP_INTERFACE_TYPE_of() - Get the interface type
+ * @interface: NFP Interface ID
+ * Return: NFP Interface ID's type
+ */
+#define NFP_CPP_INTERFACE_TYPE_of(interface) (((interface) >> 12) & 0xf)
+
+/**
+ * NFP_CPP_INTERFACE_UNIT_of() - Get the interface unit
+ * @interface: NFP Interface ID
+ * Return: NFP Interface ID's unit
+ */
+#define NFP_CPP_INTERFACE_UNIT_of(interface) (((interface) >> 8) & 0xf)
+
+/**
+ * NFP_CPP_INTERFACE_CHANNEL_of() - Get the interface channel
+ * @interface: NFP Interface ID
+ * Return: NFP Interface ID's channel
+ */
+#define NFP_CPP_INTERFACE_CHANNEL_of(interface) (((interface) >> 0) & 0xff)
+
+/* Implemented in nfp_cppcore.c */
+void nfp_cpp_free(struct nfp_cpp *cpp);
+u32 nfp_cpp_model(struct nfp_cpp *cpp);
+u16 nfp_cpp_interface(struct nfp_cpp *cpp);
+int nfp_cpp_serial(struct nfp_cpp *cpp, const u8 **serial);
+
+struct nfp_cpp_area *nfp_cpp_area_alloc_with_name(struct nfp_cpp *cpp,
+ u32 cpp_id,
+ const char *name,
+ unsigned long long address,
+ unsigned long size);
+struct nfp_cpp_area *nfp_cpp_area_alloc(struct nfp_cpp *cpp, u32 cpp_id,
+ unsigned long long address,
+ unsigned long size);
+void nfp_cpp_area_free(struct nfp_cpp_area *area);
+int nfp_cpp_area_acquire(struct nfp_cpp_area *area);
+int nfp_cpp_area_acquire_nonblocking(struct nfp_cpp_area *area);
+void nfp_cpp_area_release(struct nfp_cpp_area *area);
+void nfp_cpp_area_release_free(struct nfp_cpp_area *area);
+int nfp_cpp_area_read(struct nfp_cpp_area *area, unsigned long offset,
+ void *buffer, size_t length);
+int nfp_cpp_area_write(struct nfp_cpp_area *area, unsigned long offset,
+ const void *buffer, size_t length);
+int nfp_cpp_area_check_range(struct nfp_cpp_area *area,
+ unsigned long long offset, unsigned long size);
+const char *nfp_cpp_area_name(struct nfp_cpp_area *cpp_area);
+void *nfp_cpp_area_priv(struct nfp_cpp_area *cpp_area);
+struct nfp_cpp *nfp_cpp_area_cpp(struct nfp_cpp_area *cpp_area);
+struct resource *nfp_cpp_area_resource(struct nfp_cpp_area *area);
+phys_addr_t nfp_cpp_area_phys(struct nfp_cpp_area *area);
+void __iomem *nfp_cpp_area_iomem(struct nfp_cpp_area *area);
+
+int nfp_cpp_area_readl(struct nfp_cpp_area *area, unsigned long offset,
+ u32 *value);
+int nfp_cpp_area_writel(struct nfp_cpp_area *area, unsigned long offset,
+ u32 value);
+int nfp_cpp_area_readq(struct nfp_cpp_area *area, unsigned long offset,
+ u64 *value);
+int nfp_cpp_area_writeq(struct nfp_cpp_area *area, unsigned long offset,
+ u64 value);
+int nfp_cpp_area_fill(struct nfp_cpp_area *area, unsigned long offset,
+ u32 value, size_t length);
+
+int nfp_xpb_readl(struct nfp_cpp *cpp, u32 xpb_tgt, u32 *value);
+int nfp_xpb_writel(struct nfp_cpp *cpp, u32 xpb_tgt, u32 value);
+int nfp_xpb_writelm(struct nfp_cpp *cpp, u32 xpb_tgt, u32 mask, u32 value);
+
+/* Implemented in nfp_cpplib.c */
+int nfp_cpp_read(struct nfp_cpp *cpp, u32 cpp_id,
+ unsigned long long address, void *kernel_vaddr, size_t length);
+int nfp_cpp_write(struct nfp_cpp *cpp, u32 cpp_id,
+ unsigned long long address, const void *kernel_vaddr,
+ size_t length);
+int nfp_cpp_readl(struct nfp_cpp *cpp, u32 cpp_id,
+ unsigned long long address, u32 *value);
+int nfp_cpp_writel(struct nfp_cpp *cpp, u32 cpp_id,
+ unsigned long long address, u32 value);
+int nfp_cpp_readq(struct nfp_cpp *cpp, u32 cpp_id,
+ unsigned long long address, u64 *value);
+int nfp_cpp_writeq(struct nfp_cpp *cpp, u32 cpp_id,
+ unsigned long long address, u64 value);
+
+struct nfp_cpp_mutex;
+
+int nfp_cpp_mutex_init(struct nfp_cpp *cpp, int target,
+ unsigned long long address, u32 key_id);
+struct nfp_cpp_mutex *nfp_cpp_mutex_alloc(struct nfp_cpp *cpp, int target,
+ unsigned long long address,
+ u32 key_id);
+void nfp_cpp_mutex_free(struct nfp_cpp_mutex *mutex);
+int nfp_cpp_mutex_lock(struct nfp_cpp_mutex *mutex);
+int nfp_cpp_mutex_unlock(struct nfp_cpp_mutex *mutex);
+int nfp_cpp_mutex_trylock(struct nfp_cpp_mutex *mutex);
+
+struct nfp_cpp_explicit;
+
+struct nfp_cpp_explicit_command {
+ u32 cpp_id;
+ u16 data_ref;
+ u8 data_master;
+ u8 len;
+ u8 byte_mask;
+ u8 signal_master;
+ u8 signal_ref;
+ u8 posted;
+ u8 siga;
+ u8 sigb;
+ s8 siga_mode;
+ s8 sigb_mode;
+};
+
+#define NFP_SERIAL_LEN 6
+
+/**
+ * struct nfp_cpp_operations - NFP CPP operations structure
+ * @area_priv_size: Size of the nfp_cpp_area private data
+ * @owner: Owner module
+ * @init: Initialize the NFP CPP bus
+ * @free: Free the bus
+ * @read_serial: Read serial number to memory provided
+ * @get_interface: Return CPP interface
+ * @area_init: Initialize a new NFP CPP area (not serialized)
+ * @area_cleanup: Clean up a NFP CPP area (not serialized)
+ * @area_acquire: Acquire the NFP CPP area (serialized)
+ * @area_release: Release area (serialized)
+ * @area_resource: Get resource range of area (not serialized)
+ * @area_phys: Get physical address of area (not serialized)
+ * @area_iomem: Get iomem of area (not serialized)
+ * @area_read: Perform a read from a NFP CPP area (serialized)
+ * @area_write: Perform a write to a NFP CPP area (serialized)
+ * @explicit_priv_size: Size of an explicit's private area
+ * @explicit_acquire: Acquire an explicit area
+ * @explicit_release: Release an explicit area
+ * @explicit_put: Write data to send
+ * @explicit_get: Read data received
+ * @explicit_do: Perform the transaction
+ */
+struct nfp_cpp_operations {
+ size_t area_priv_size;
+ struct module *owner;
+
+ int (*init)(struct nfp_cpp *cpp);
+ void (*free)(struct nfp_cpp *cpp);
+
+ void (*read_serial)(struct device *dev, u8 *serial);
+ u16 (*get_interface)(struct device *dev);
+
+ int (*area_init)(struct nfp_cpp_area *area,
+ u32 dest, unsigned long long address,
+ unsigned long size);
+ void (*area_cleanup)(struct nfp_cpp_area *area);
+ int (*area_acquire)(struct nfp_cpp_area *area);
+ void (*area_release)(struct nfp_cpp_area *area);
+ struct resource *(*area_resource)(struct nfp_cpp_area *area);
+ phys_addr_t (*area_phys)(struct nfp_cpp_area *area);
+ void __iomem *(*area_iomem)(struct nfp_cpp_area *area);
+ int (*area_read)(struct nfp_cpp_area *area, void *kernel_vaddr,
+ unsigned long offset, unsigned int length);
+ int (*area_write)(struct nfp_cpp_area *area, const void *kernel_vaddr,
+ unsigned long offset, unsigned int length);
+
+ size_t explicit_priv_size;
+ int (*explicit_acquire)(struct nfp_cpp_explicit *expl);
+ void (*explicit_release)(struct nfp_cpp_explicit *expl);
+ int (*explicit_put)(struct nfp_cpp_explicit *expl,
+ const void *buff, size_t len);
+ int (*explicit_get)(struct nfp_cpp_explicit *expl,
+ void *buff, size_t len);
+ int (*explicit_do)(struct nfp_cpp_explicit *expl,
+ const struct nfp_cpp_explicit_command *cmd,
+ u64 address);
+};
+
+struct nfp_cpp *
+nfp_cpp_from_operations(const struct nfp_cpp_operations *ops,
+ struct device *parent, void *priv);
+void *nfp_cpp_priv(struct nfp_cpp *priv);
+
+int nfp_cpp_area_cache_add(struct nfp_cpp *cpp, size_t size);
+
+/* The following section contains extensions to the
+ * NFP CPP API, to be used in a Linux kernel-space context.
+ */
+
+/* Use this channel ID for multiple virtual channel interfaces
+ * (ie ARM and PCIe) when setting up the interface field.
+ */
+#define NFP_CPP_INTERFACE_CHANNEL_PEROPENER 255
+struct device *nfp_cpp_device(struct nfp_cpp *cpp);
+
+/* Return code masks for nfp_cpp_explicit_do()
+ */
+#define NFP_SIGNAL_MASK_A BIT(0) /* Signal A fired */
+#define NFP_SIGNAL_MASK_B BIT(1) /* Signal B fired */
+
+enum nfp_cpp_explicit_signal_mode {
+ NFP_SIGNAL_NONE = 0,
+ NFP_SIGNAL_PUSH = 1,
+ NFP_SIGNAL_PUSH_OPTIONAL = -1,
+ NFP_SIGNAL_PULL = 2,
+ NFP_SIGNAL_PULL_OPTIONAL = -2,
+};
+
+struct nfp_cpp_explicit *nfp_cpp_explicit_acquire(struct nfp_cpp *cpp);
+int nfp_cpp_explicit_set_target(struct nfp_cpp_explicit *expl, u32 cpp_id,
+ u8 len, u8 mask);
+int nfp_cpp_explicit_set_data(struct nfp_cpp_explicit *expl,
+ u8 data_master, u16 data_ref);
+int nfp_cpp_explicit_set_signal(struct nfp_cpp_explicit *expl,
+ u8 signal_master, u8 signal_ref);
+int nfp_cpp_explicit_set_posted(struct nfp_cpp_explicit *expl, int posted,
+ u8 siga,
+ enum nfp_cpp_explicit_signal_mode siga_mode,
+ u8 sigb,
+ enum nfp_cpp_explicit_signal_mode sigb_mode);
+int nfp_cpp_explicit_put(struct nfp_cpp_explicit *expl,
+ const void *buff, size_t len);
+int nfp_cpp_explicit_do(struct nfp_cpp_explicit *expl, u64 address);
+int nfp_cpp_explicit_get(struct nfp_cpp_explicit *expl, void *buff, size_t len);
+void nfp_cpp_explicit_release(struct nfp_cpp_explicit *expl);
+struct nfp_cpp *nfp_cpp_explicit_cpp(struct nfp_cpp_explicit *expl);
+void *nfp_cpp_explicit_priv(struct nfp_cpp_explicit *cpp_explicit);
+
+/* Implemented in nfp_cpplib.c */
+
+int nfp_cpp_model_autodetect(struct nfp_cpp *cpp, u32 *model);
+
+int nfp_cpp_explicit_read(struct nfp_cpp *cpp, u32 cpp_id,
+ u64 addr, void *buff, size_t len,
+ int width_read);
+
+int nfp_cpp_explicit_write(struct nfp_cpp *cpp, u32 cpp_id,
+ u64 addr, const void *buff, size_t len,
+ int width_write);
+
+#endif /* !__NFP_CPP_H__ */
--- /dev/null
+/*
+ * Copyright (C) 2015-2017 Netronome Systems, Inc.
+ *
+ * This software is dual licensed under the GNU General License Version 2,
+ * June 1991 as shown in the file COPYING in the top-level directory of this
+ * source tree or the BSD 2-Clause License provided below. You have the
+ * option to license this software under the complete terms of either license.
+ *
+ * The BSD 2-Clause License:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+/*
+ * nfp_cppcore.c
+ * Provides low-level access to the NFP's internal CPP bus
+ * Authors: Jakub Kicinski <jakub.kicinski@netronome.com>
+ * Jason McMullan <jason.mcmullan@netronome.com>
+ * Rolf Neugebauer <rolf.neugebauer@netronome.com>
+ */
+
+#include <asm/unaligned.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/ioport.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/wait.h>
+
+#include "nfp_arm.h"
+#include "nfp_cpp.h"
+#include "nfp6000/nfp6000.h"
+
+#define NFP_ARM_GCSR_SOFTMODEL2 0x0000014c
+#define NFP_ARM_GCSR_SOFTMODEL3 0x00000150
+
+struct nfp_cpp_resource {
+ struct list_head list;
+ const char *name;
+ u32 cpp_id;
+ u64 start;
+ u64 end;
+};
+
+struct nfp_cpp_mutex {
+ struct list_head list;
+ struct nfp_cpp *cpp;
+ int target;
+ u16 usage;
+ u16 depth;
+ unsigned long long address;
+ u32 key;
+};
+
+struct nfp_cpp {
+ struct device dev;
+
+ void *priv; /* Private data of the low-level implementation */
+
+ u32 model;
+ u16 interface;
+ u8 serial[NFP_SERIAL_LEN];
+
+ const struct nfp_cpp_operations *op;
+ struct list_head resource_list; /* NFP CPP resource list */
+ struct list_head mutex_cache; /* Mutex cache */
+ rwlock_t resource_lock;
+ wait_queue_head_t waitq;
+
+ /* NFP6000 CPP Mapping Table */
+ u32 imb_cat_table[16];
+
+ /* Cached areas for cpp/xpb readl/writel speedups */
+ struct mutex area_cache_mutex; /* Lock for the area cache */
+ struct list_head area_cache_list;
+};
+
+/* Element of the area_cache_list */
+struct nfp_cpp_area_cache {
+ struct list_head entry;
+ u32 id;
+ u64 addr;
+ u32 size;
+ struct nfp_cpp_area *area;
+};
+
+struct nfp_cpp_area {
+ struct nfp_cpp *cpp;
+ struct kref kref;
+ atomic_t refcount;
+ struct mutex mutex; /* Lock for the area's refcount */
+ unsigned long long offset;
+ unsigned long size;
+ struct nfp_cpp_resource resource;
+ void __iomem *iomem;
+ /* Here follows the 'priv' part of nfp_cpp_area. */
+};
+
+struct nfp_cpp_explicit {
+ struct nfp_cpp *cpp;
+ struct nfp_cpp_explicit_command cmd;
+ /* Here follows the 'priv' part of nfp_cpp_area. */
+};
+
+static void __resource_add(struct list_head *head, struct nfp_cpp_resource *res)
+{
+ struct nfp_cpp_resource *tmp;
+ struct list_head *pos;
+
+ list_for_each(pos, head) {
+ tmp = container_of(pos, struct nfp_cpp_resource, list);
+
+ if (tmp->cpp_id > res->cpp_id)
+ break;
+
+ if (tmp->cpp_id == res->cpp_id && tmp->start > res->start)
+ break;
+ }
+
+ list_add_tail(&res->list, pos);
+}
+
+static void __resource_del(struct nfp_cpp_resource *res)
+{
+ list_del_init(&res->list);
+}
+
+static void __release_cpp_area(struct kref *kref)
+{
+ struct nfp_cpp_area *area =
+ container_of(kref, struct nfp_cpp_area, kref);
+ struct nfp_cpp *cpp = nfp_cpp_area_cpp(area);
+
+ if (area->cpp->op->area_cleanup)
+ area->cpp->op->area_cleanup(area);
+
+ write_lock(&cpp->resource_lock);
+ __resource_del(&area->resource);
+ write_unlock(&cpp->resource_lock);
+ kfree(area);
+}
+
+static void nfp_cpp_area_put(struct nfp_cpp_area *area)
+{
+ kref_put(&area->kref, __release_cpp_area);
+}
+
+static struct nfp_cpp_area *nfp_cpp_area_get(struct nfp_cpp_area *area)
+{
+ kref_get(&area->kref);
+
+ return area;
+}
+
+/**
+ * nfp_cpp_free() - free the CPP handle
+ * @cpp: CPP handle
+ */
+void nfp_cpp_free(struct nfp_cpp *cpp)
+{
+ struct nfp_cpp_area_cache *cache, *ctmp;
+ struct nfp_cpp_resource *res, *rtmp;
+ struct nfp_cpp_mutex *mutex, *mtmp;
+
+ /* There should be no mutexes in the cache at this point. */
+ WARN_ON(!list_empty(&cpp->mutex_cache));
+ /* .. but if there are, unlock them and complain. */
+ list_for_each_entry_safe(mutex, mtmp, &cpp->mutex_cache, list) {
+ dev_err(cpp->dev.parent, "Dangling mutex: @%d::0x%llx, %d locks held by %d owners\n",
+ mutex->target, (unsigned long long)mutex->address,
+ mutex->depth, mutex->usage);
+
+ /* Forcing an unlock */
+ mutex->depth = 1;
+ nfp_cpp_mutex_unlock(mutex);
+
+ /* Forcing a free */
+ mutex->usage = 1;
+ nfp_cpp_mutex_free(mutex);
+ }
+
+ /* Remove all caches */
+ list_for_each_entry_safe(cache, ctmp, &cpp->area_cache_list, entry) {
+ list_del(&cache->entry);
+ if (cache->id)
+ nfp_cpp_area_release(cache->area);
+ nfp_cpp_area_free(cache->area);
+ kfree(cache);
+ }
+
+ /* There should be no dangling areas at this point */
+ WARN_ON(!list_empty(&cpp->resource_list));
+
+ /* .. but if they weren't, try to clean up. */
+ list_for_each_entry_safe(res, rtmp, &cpp->resource_list, list) {
+ struct nfp_cpp_area *area = container_of(res,
+ struct nfp_cpp_area,
+ resource);
+
+ dev_err(cpp->dev.parent, "Dangling area: %d:%d:%d:0x%0llx-0x%0llx%s%s\n",
+ NFP_CPP_ID_TARGET_of(res->cpp_id),
+ NFP_CPP_ID_ACTION_of(res->cpp_id),
+ NFP_CPP_ID_TOKEN_of(res->cpp_id),
+ res->start, res->end,
+ res->name ? " " : "",
+ res->name ? res->name : "");
+
+ if (area->cpp->op->area_release)
+ area->cpp->op->area_release(area);
+
+ __release_cpp_area(&area->kref);
+ }
+
+ if (cpp->op->free)
+ cpp->op->free(cpp);
+
+ device_unregister(&cpp->dev);
+
+ kfree(cpp);
+}
+
+/**
+ * nfp_cpp_model() - Retrieve the Model ID of the NFP
+ * @cpp: NFP CPP handle
+ *
+ * Return: NFP CPP Model ID
+ */
+u32 nfp_cpp_model(struct nfp_cpp *cpp)
+{
+ return cpp->model;
+}
+
+/**
+ * nfp_cpp_interface() - Retrieve the Interface ID of the NFP
+ * @cpp: NFP CPP handle
+ *
+ * Return: NFP CPP Interface ID
+ */
+u16 nfp_cpp_interface(struct nfp_cpp *cpp)
+{
+ return cpp->interface;
+}
+
+/**
+ * nfp_cpp_serial() - Retrieve the Serial ID of the NFP
+ * @cpp: NFP CPP handle
+ * @serial: Pointer to NFP serial number
+ *
+ * Return: Length of NFP serial number
+ */
+int nfp_cpp_serial(struct nfp_cpp *cpp, const u8 **serial)
+{
+ *serial = &cpp->serial[0];
+ return sizeof(cpp->serial);
+}
+
+/**
+ * nfp_cpp_area_alloc_with_name() - allocate a new CPP area
+ * @cpp: CPP device handle
+ * @dest: NFP CPP ID
+ * @name: Name of region
+ * @address: Address of region
+ * @size: Size of region
+ *
+ * Allocate and initialize a CPP area structure. The area must later
+ * be locked down with an 'acquire' before it can be safely accessed.
+ *
+ * NOTE: @address and @size must be 32-bit aligned values.
+ *
+ * Return: NFP CPP area handle, or NULL
+ */
+struct nfp_cpp_area *
+nfp_cpp_area_alloc_with_name(struct nfp_cpp *cpp, u32 dest, const char *name,
+ unsigned long long address, unsigned long size)
+{
+ struct nfp_cpp_area *area;
+ u64 tmp64 = address;
+ int err, name_len;
+
+ /* Remap from cpp_island to cpp_target */
+ err = nfp_target_cpp(dest, tmp64, &dest, &tmp64, cpp->imb_cat_table);
+ if (err < 0)
+ return NULL;
+
+ address = tmp64;
+
+ if (!name)
+ name = "(reserved)";
+
+ name_len = strlen(name) + 1;
+ area = kzalloc(sizeof(*area) + cpp->op->area_priv_size + name_len,
+ GFP_KERNEL);
+ if (!area)
+ return NULL;
+
+ area->cpp = cpp;
+ area->resource.name = (void *)area + sizeof(*area) +
+ cpp->op->area_priv_size;
+ memcpy((char *)area->resource.name, name, name_len);
+
+ area->resource.cpp_id = dest;
+ area->resource.start = address;
+ area->resource.end = area->resource.start + size - 1;
+ INIT_LIST_HEAD(&area->resource.list);
+
+ atomic_set(&area->refcount, 0);
+ kref_init(&area->kref);
+ mutex_init(&area->mutex);
+
+ if (cpp->op->area_init) {
+ int err;
+
+ err = cpp->op->area_init(area, dest, address, size);
+ if (err < 0) {
+ kfree(area);
+ return NULL;
+ }
+ }
+
+ write_lock(&cpp->resource_lock);
+ __resource_add(&cpp->resource_list, &area->resource);
+ write_unlock(&cpp->resource_lock);
+
+ area->offset = address;
+ area->size = size;
+
+ return area;
+}
+
+/**
+ * nfp_cpp_area_alloc() - allocate a new CPP area
+ * @cpp: CPP handle
+ * @dest: CPP id
+ * @address: Start address on CPP target
+ * @size: Size of area in bytes
+ *
+ * Allocate and initialize a CPP area structure. The area must later
+ * be locked down with an 'acquire' before it can be safely accessed.
+ *
+ * NOTE: @address and @size must be 32-bit aligned values.
+ *
+ * Return: NFP CPP Area handle, or NULL
+ */
+struct nfp_cpp_area *
+nfp_cpp_area_alloc(struct nfp_cpp *cpp, u32 dest,
+ unsigned long long address, unsigned long size)
+{
+ return nfp_cpp_area_alloc_with_name(cpp, dest, NULL, address, size);
+}
+
+/**
+ * nfp_cpp_area_free() - free up the CPP area
+ * @area: CPP area handle
+ *
+ * Frees up memory resources held by the CPP area.
+ */
+void nfp_cpp_area_free(struct nfp_cpp_area *area)
+{
+ nfp_cpp_area_put(area);
+}
+
+/**
+ * nfp_cpp_area_acquire() - lock down a CPP area for access
+ * @area: CPP area handle
+ *
+ * Locks down the CPP area for a potential long term activity. Area
+ * must always be locked down before being accessed.
+ *
+ * Return: 0, or -ERRNO
+ */
+int nfp_cpp_area_acquire(struct nfp_cpp_area *area)
+{
+ mutex_lock(&area->mutex);
+ if (atomic_inc_return(&area->refcount) == 1) {
+ int (*a_a)(struct nfp_cpp_area *);
+
+ a_a = area->cpp->op->area_acquire;
+ if (a_a) {
+ int err;
+
+ wait_event_interruptible(area->cpp->waitq,
+ (err = a_a(area)) != -EAGAIN);
+ if (err < 0) {
+ atomic_dec(&area->refcount);
+ mutex_unlock(&area->mutex);
+ return err;
+ }
+ }
+ }
+ mutex_unlock(&area->mutex);
+
+ nfp_cpp_area_get(area);
+ return 0;
+}
+
+/**
+ * nfp_cpp_area_acquire_nonblocking() - lock down a CPP area for access
+ * @area: CPP area handle
+ *
+ * Locks down the CPP area for a potential long term activity. Area
+ * must always be locked down before being accessed.
+ *
+ * NOTE: Returns -EAGAIN is no area is available
+ *
+ * Return: 0, or -ERRNO
+ */
+int nfp_cpp_area_acquire_nonblocking(struct nfp_cpp_area *area)
+{
+ mutex_lock(&area->mutex);
+ if (atomic_inc_return(&area->refcount) == 1) {
+ if (area->cpp->op->area_acquire) {
+ int err;
+
+ err = area->cpp->op->area_acquire(area);
+ if (err < 0) {
+ atomic_dec(&area->refcount);
+ mutex_unlock(&area->mutex);
+ return err;
+ }
+ }
+ }
+ mutex_unlock(&area->mutex);
+
+ nfp_cpp_area_get(area);
+ return 0;
+}
+
+/**
+ * nfp_cpp_area_release() - release a locked down CPP area
+ * @area: CPP area handle
+ *
+ * Releases a previously locked down CPP area.
+ */
+void nfp_cpp_area_release(struct nfp_cpp_area *area)
+{
+ mutex_lock(&area->mutex);
+ /* Only call the release on refcount == 0 */
+ if (atomic_dec_and_test(&area->refcount)) {
+ if (area->cpp->op->area_release) {
+ area->cpp->op->area_release(area);
+ /* Let anyone waiting for a BAR try to get one.. */
+ wake_up_interruptible_all(&area->cpp->waitq);
+ }
+ }
+ mutex_unlock(&area->mutex);
+
+ nfp_cpp_area_put(area);
+}
+
+/**
+ * nfp_cpp_area_release_free() - release CPP area and free it
+ * @area: CPP area handle
+ *
+ * Releases CPP area and frees up memory resources held by the it.
+ */
+void nfp_cpp_area_release_free(struct nfp_cpp_area *area)
+{
+ nfp_cpp_area_release(area);
+ nfp_cpp_area_free(area);
+}
+
+/**
+ * nfp_cpp_area_read() - read data from CPP area
+ * @area: CPP area handle
+ * @offset: offset into CPP area
+ * @kernel_vaddr: kernel address to put data into
+ * @length: number of bytes to read
+ *
+ * Read data from indicated CPP region.
+ *
+ * NOTE: @offset and @length must be 32-bit aligned values.
+ *
+ * NOTE: Area must have been locked down with an 'acquire'.
+ *
+ * Return: length of io, or -ERRNO
+ */
+int nfp_cpp_area_read(struct nfp_cpp_area *area,
+ unsigned long offset, void *kernel_vaddr,
+ size_t length)
+{
+ return area->cpp->op->area_read(area, kernel_vaddr, offset, length);
+}
+
+/**
+ * nfp_cpp_area_write() - write data to CPP area
+ * @area: CPP area handle
+ * @offset: offset into CPP area
+ * @kernel_vaddr: kernel address to read data from
+ * @length: number of bytes to write
+ *
+ * Write data to indicated CPP region.
+ *
+ * NOTE: @offset and @length must be 32-bit aligned values.
+ *
+ * NOTE: Area must have been locked down with an 'acquire'.
+ *
+ * Return: length of io, or -ERRNO
+ */
+int nfp_cpp_area_write(struct nfp_cpp_area *area,
+ unsigned long offset, const void *kernel_vaddr,
+ size_t length)
+{
+ return area->cpp->op->area_write(area, kernel_vaddr, offset, length);
+}
+
+/**
+ * nfp_cpp_area_check_range() - check if address range fits in CPP area
+ * @area: CPP area handle
+ * @offset: offset into CPP target
+ * @length: size of address range in bytes
+ *
+ * Check if address range fits within CPP area. Return 0 if area
+ * fits or -EFAULT on error.
+ *
+ * Return: 0, or -ERRNO
+ */
+int nfp_cpp_area_check_range(struct nfp_cpp_area *area,
+ unsigned long long offset, unsigned long length)
+{
+ if (offset < area->offset ||
+ offset + length > area->offset + area->size)
+ return -EFAULT;
+
+ return 0;
+}
+
+/**
+ * nfp_cpp_area_name() - return name of a CPP area
+ * @cpp_area: CPP area handle
+ *
+ * Return: Name of the area, or NULL
+ */
+const char *nfp_cpp_area_name(struct nfp_cpp_area *cpp_area)
+{
+ return cpp_area->resource.name;
+}
+
+/**
+ * nfp_cpp_area_priv() - return private struct for CPP area
+ * @cpp_area: CPP area handle
+ *
+ * Return: Private data for the CPP area
+ */
+void *nfp_cpp_area_priv(struct nfp_cpp_area *cpp_area)
+{
+ return &cpp_area[1];
+}
+
+/**
+ * nfp_cpp_area_cpp() - return CPP handle for CPP area
+ * @cpp_area: CPP area handle
+ *
+ * Return: NFP CPP handle
+ */
+struct nfp_cpp *nfp_cpp_area_cpp(struct nfp_cpp_area *cpp_area)
+{
+ return cpp_area->cpp;
+}
+
+/**
+ * nfp_cpp_area_resource() - get resource
+ * @area: CPP area handle
+ *
+ * NOTE: Area must have been locked down with an 'acquire'.
+ *
+ * Return: struct resource pointer, or NULL
+ */
+struct resource *nfp_cpp_area_resource(struct nfp_cpp_area *area)
+{
+ struct resource *res = NULL;
+
+ if (area->cpp->op->area_resource)
+ res = area->cpp->op->area_resource(area);
+
+ return res;
+}
+
+/**
+ * nfp_cpp_area_phys() - get physical address of CPP area
+ * @area: CPP area handle
+ *
+ * NOTE: Area must have been locked down with an 'acquire'.
+ *
+ * Return: phy_addr_t of the area, or NULL
+ */
+phys_addr_t nfp_cpp_area_phys(struct nfp_cpp_area *area)
+{
+ phys_addr_t addr = ~0;
+
+ if (area->cpp->op->area_phys)
+ addr = area->cpp->op->area_phys(area);
+
+ return addr;
+}
+
+/**
+ * nfp_cpp_area_iomem() - get IOMEM region for CPP area
+ * @area: CPP area handle
+ *
+ * Returns an iomem pointer for use with readl()/writel() style
+ * operations.
+ *
+ * NOTE: Area must have been locked down with an 'acquire'.
+ *
+ * Return: __iomem pointer to the area, or NULL
+ */
+void __iomem *nfp_cpp_area_iomem(struct nfp_cpp_area *area)
+{
+ void __iomem *iomem = NULL;
+
+ if (area->cpp->op->area_iomem)
+ iomem = area->cpp->op->area_iomem(area);
+
+ return iomem;
+}
+
+/**
+ * nfp_cpp_area_readl() - Read a u32 word from an area
+ * @area: CPP Area handle
+ * @offset: Offset into area
+ * @value: Pointer to read buffer
+ *
+ * Return: length of the io, or -ERRNO
+ */
+int nfp_cpp_area_readl(struct nfp_cpp_area *area,
+ unsigned long offset, u32 *value)
+{
+ u8 tmp[4];
+ int err;
+
+ err = nfp_cpp_area_read(area, offset, &tmp, sizeof(tmp));
+ *value = get_unaligned_le32(tmp);
+
+ return err;
+}
+
+/**
+ * nfp_cpp_area_writel() - Write a u32 word to an area
+ * @area: CPP Area handle
+ * @offset: Offset into area
+ * @value: Value to write
+ *
+ * Return: length of the io, or -ERRNO
+ */
+int nfp_cpp_area_writel(struct nfp_cpp_area *area,
+ unsigned long offset, u32 value)
+{
+ u8 tmp[4];
+
+ put_unaligned_le32(value, tmp);
+
+ return nfp_cpp_area_write(area, offset, &tmp, sizeof(tmp));
+}
+
+/**
+ * nfp_cpp_area_readq() - Read a u64 word from an area
+ * @area: CPP Area handle
+ * @offset: Offset into area
+ * @value: Pointer to read buffer
+ *
+ * Return: length of the io, or -ERRNO
+ */
+int nfp_cpp_area_readq(struct nfp_cpp_area *area,
+ unsigned long offset, u64 *value)
+{
+ u8 tmp[8];
+ int err;
+
+ err = nfp_cpp_area_read(area, offset, &tmp, sizeof(tmp));
+ *value = get_unaligned_le64(tmp);
+
+ return err;
+}
+
+/**
+ * nfp_cpp_area_writeq() - Write a u64 word to an area
+ * @area: CPP Area handle
+ * @offset: Offset into area
+ * @value: Value to write
+ *
+ * Return: length of the io, or -ERRNO
+ */
+int nfp_cpp_area_writeq(struct nfp_cpp_area *area,
+ unsigned long offset, u64 value)
+{
+ u8 tmp[8];
+
+ put_unaligned_le64(value, tmp);
+
+ return nfp_cpp_area_write(area, offset, &tmp, sizeof(tmp));
+}
+
+/**
+ * nfp_cpp_area_fill() - fill a CPP area with a value
+ * @area: CPP area
+ * @offset: offset into CPP area
+ * @value: value to fill with
+ * @length: length of area to fill
+ *
+ * Fill indicated area with given value.
+ *
+ * Return: length of io, or -ERRNO
+ */
+int nfp_cpp_area_fill(struct nfp_cpp_area *area,
+ unsigned long offset, u32 value, size_t length)
+{
+ u8 tmp[4];
+ size_t i;
+ int k;
+
+ put_unaligned_le32(value, tmp);
+
+ if (offset % sizeof(tmp) || length % sizeof(tmp))
+ return -EINVAL;
+
+ for (i = 0; i < length; i += sizeof(tmp)) {
+ k = nfp_cpp_area_write(area, offset + i, &tmp, sizeof(tmp));
+ if (k < 0)
+ return k;
+ }
+
+ return i;
+}
+
+/**
+ * nfp_cpp_area_cache_add() - Permanently reserve and area for the hot cache
+ * @cpp: NFP CPP handle
+ * @size: Size of the area - MUST BE A POWER OF 2.
+ */
+int nfp_cpp_area_cache_add(struct nfp_cpp *cpp, size_t size)
+{
+ struct nfp_cpp_area_cache *cache;
+ struct nfp_cpp_area *area;
+
+ /* Allocate an area - we use the MU target's base as a placeholder,
+ * as all supported chips have a MU.
+ */
+ area = nfp_cpp_area_alloc(cpp, NFP_CPP_ID(7, NFP_CPP_ACTION_RW, 0),
+ 0, size);
+ if (!area)
+ return -ENOMEM;
+
+ cache = kzalloc(sizeof(*cache), GFP_KERNEL);
+ if (!cache)
+ return -ENOMEM;
+
+ cache->id = 0;
+ cache->addr = 0;
+ cache->size = size;
+ cache->area = area;
+ mutex_lock(&cpp->area_cache_mutex);
+ list_add_tail(&cache->entry, &cpp->area_cache_list);
+ mutex_unlock(&cpp->area_cache_mutex);
+
+ return 0;
+}
+
+static struct nfp_cpp_area_cache *
+area_cache_get(struct nfp_cpp *cpp, u32 id,
+ u64 addr, unsigned long *offset, size_t length)
+{
+ struct nfp_cpp_area_cache *cache;
+ int err;
+
+ /* Early exit when length == 0, which prevents
+ * the need for special case code below when
+ * checking against available cache size.
+ */
+ if (length == 0)
+ return NULL;
+
+ if (list_empty(&cpp->area_cache_list) || id == 0)
+ return NULL;
+
+ /* Remap from cpp_island to cpp_target */
+ err = nfp_target_cpp(id, addr, &id, &addr, cpp->imb_cat_table);
+ if (err < 0)
+ return NULL;
+
+ addr += *offset;
+
+ mutex_lock(&cpp->area_cache_mutex);
+
+ /* See if we have a match */
+ list_for_each_entry(cache, &cpp->area_cache_list, entry) {
+ if (id == cache->id &&
+ addr >= cache->addr &&
+ addr + length <= cache->addr + cache->size)
+ goto exit;
+ }
+
+ /* No matches - inspect the tail of the LRU */
+ cache = list_entry(cpp->area_cache_list.prev,
+ struct nfp_cpp_area_cache, entry);
+
+ /* Can we fit in the cache entry? */
+ if (round_down(addr + length - 1, cache->size) !=
+ round_down(addr, cache->size)) {
+ mutex_unlock(&cpp->area_cache_mutex);
+ return NULL;
+ }
+
+ /* If id != 0, we will need to release it */
+ if (cache->id) {
+ nfp_cpp_area_release(cache->area);
+ cache->id = 0;
+ cache->addr = 0;
+ }
+
+ /* Adjust the start address to be cache size aligned */
+ cache->id = id;
+ cache->addr = addr & ~(u64)(cache->size - 1);
+
+ /* Re-init to the new ID and address */
+ if (cpp->op->area_init) {
+ err = cpp->op->area_init(cache->area,
+ id, cache->addr, cache->size);
+ if (err < 0) {
+ mutex_unlock(&cpp->area_cache_mutex);
+ return NULL;
+ }
+ }
+
+ /* Attempt to acquire */
+ err = nfp_cpp_area_acquire(cache->area);
+ if (err < 0) {
+ mutex_unlock(&cpp->area_cache_mutex);
+ return NULL;
+ }
+
+exit:
+ /* Adjust offset */
+ *offset = addr - cache->addr;
+ return cache;
+}
+
+static void
+area_cache_put(struct nfp_cpp *cpp, struct nfp_cpp_area_cache *cache)
+{
+ if (!cache)
+ return;
+
+ /* Move to front of LRU */
+ list_del(&cache->entry);
+ list_add(&cache->entry, &cpp->area_cache_list);
+
+ mutex_unlock(&cpp->area_cache_mutex);
+}
+
+/**
+ * nfp_cpp_read() - read from CPP target
+ * @cpp: CPP handle
+ * @destination: CPP id
+ * @address: offset into CPP target
+ * @kernel_vaddr: kernel buffer for result
+ * @length: number of bytes to read
+ *
+ * Return: length of io, or -ERRNO
+ */
+int nfp_cpp_read(struct nfp_cpp *cpp, u32 destination,
+ unsigned long long address, void *kernel_vaddr, size_t length)
+{
+ struct nfp_cpp_area_cache *cache;
+ struct nfp_cpp_area *area;
+ unsigned long offset = 0;
+ int err;
+
+ cache = area_cache_get(cpp, destination, address, &offset, length);
+ if (cache) {
+ area = cache->area;
+ } else {
+ area = nfp_cpp_area_alloc(cpp, destination, address, length);
+ if (!area)
+ return -ENOMEM;
+
+ err = nfp_cpp_area_acquire(area);
+ if (err)
+ goto out;
+ }
+
+ err = nfp_cpp_area_read(area, offset, kernel_vaddr, length);
+out:
+ if (cache)
+ area_cache_put(cpp, cache);
+ else
+ nfp_cpp_area_release_free(area);
+
+ return err;
+}
+
+/**
+ * nfp_cpp_write() - write to CPP target
+ * @cpp: CPP handle
+ * @destination: CPP id
+ * @address: offset into CPP target
+ * @kernel_vaddr: kernel buffer to read from
+ * @length: number of bytes to write
+ *
+ * Return: length of io, or -ERRNO
+ */
+int nfp_cpp_write(struct nfp_cpp *cpp, u32 destination,
+ unsigned long long address,
+ const void *kernel_vaddr, size_t length)
+{
+ struct nfp_cpp_area_cache *cache;
+ struct nfp_cpp_area *area;
+ unsigned long offset = 0;
+ int err;
+
+ cache = area_cache_get(cpp, destination, address, &offset, length);
+ if (cache) {
+ area = cache->area;
+ } else {
+ area = nfp_cpp_area_alloc(cpp, destination, address, length);
+ if (!area)
+ return -ENOMEM;
+
+ err = nfp_cpp_area_acquire(area);
+ if (err)
+ goto out;
+ }
+
+ err = nfp_cpp_area_write(area, offset, kernel_vaddr, length);
+
+out:
+ if (cache)
+ area_cache_put(cpp, cache);
+ else
+ nfp_cpp_area_release_free(area);
+
+ return err;
+}
+
+/* Return the correct CPP address, and fixup xpb_addr as needed. */
+static u32 nfp_xpb_to_cpp(struct nfp_cpp *cpp, u32 *xpb_addr)
+{
+ int island;
+ u32 xpb;
+
+ xpb = NFP_CPP_ID(14, NFP_CPP_ACTION_RW, 0);
+ /* Ensure that non-local XPB accesses go
+ * out through the global XPBM bus.
+ */
+ island = (*xpb_addr >> 24) & 0x3f;
+ if (!island)
+ return xpb;
+
+ if (island != 1) {
+ *xpb_addr |= 1 << 30;
+ return xpb;
+ }
+
+ /* Accesses to the ARM Island overlay uses Island 0 / Global Bit */
+ *xpb_addr &= ~0x7f000000;
+ if (*xpb_addr < 0x60000) {
+ *xpb_addr |= 1 << 30;
+ } else {
+ /* And only non-ARM interfaces use the island id = 1 */
+ if (NFP_CPP_INTERFACE_TYPE_of(nfp_cpp_interface(cpp))
+ != NFP_CPP_INTERFACE_TYPE_ARM)
+ *xpb_addr |= 1 << 24;
+ }
+
+ return xpb;
+}
+
+/**
+ * nfp_xpb_readl() - Read a u32 word from a XPB location
+ * @cpp: CPP device handle
+ * @xpb_addr: Address for operation
+ * @value: Pointer to read buffer
+ *
+ * Return: length of the io, or -ERRNO
+ */
+int nfp_xpb_readl(struct nfp_cpp *cpp, u32 xpb_addr, u32 *value)
+{
+ u32 cpp_dest = nfp_xpb_to_cpp(cpp, &xpb_addr);
+
+ return nfp_cpp_readl(cpp, cpp_dest, xpb_addr, value);
+}
+
+/**
+ * nfp_xpb_writel() - Write a u32 word to a XPB location
+ * @cpp: CPP device handle
+ * @xpb_addr: Address for operation
+ * @value: Value to write
+ *
+ * Return: length of the io, or -ERRNO
+ */
+int nfp_xpb_writel(struct nfp_cpp *cpp, u32 xpb_addr, u32 value)
+{
+ u32 cpp_dest = nfp_xpb_to_cpp(cpp, &xpb_addr);
+
+ return nfp_cpp_writel(cpp, cpp_dest, xpb_addr, value);
+}
+
+/**
+ * nfp_xpb_writelm() - Modify bits of a 32-bit value from the XPB bus
+ * @cpp: NFP CPP device handle
+ * @xpb_tgt: XPB target and address
+ * @mask: mask of bits to alter
+ * @value: value to modify
+ *
+ * KERNEL: This operation is safe to call in interrupt or softirq context.
+ *
+ * Return: length of the io, or -ERRNO
+ */
+int nfp_xpb_writelm(struct nfp_cpp *cpp, u32 xpb_tgt,
+ u32 mask, u32 value)
+{
+ int err;
+ u32 tmp;
+
+ err = nfp_xpb_readl(cpp, xpb_tgt, &tmp);
+ if (err < 0)
+ return err;
+
+ tmp &= ~mask;
+ tmp |= mask & value;
+ return nfp_xpb_writel(cpp, xpb_tgt, tmp);
+}
+
+/* Lockdep markers */
+static struct lock_class_key nfp_cpp_resource_lock_key;
+
+static void nfp_cpp_dev_release(struct device *dev)
+{
+ /* Nothing to do here - it just makes the kernel happy */
+}
+
+/**
+ * nfp_cpp_from_operations() - Create a NFP CPP handle
+ * from an operations structure
+ * @ops: NFP CPP operations structure
+ * @parent: Parent device
+ * @priv: Private data of low-level implementation
+ *
+ * NOTE: On failure, cpp_ops->free will be called!
+ *
+ * Return: NFP CPP handle on success, ERR_PTR on failure
+ */
+struct nfp_cpp *
+nfp_cpp_from_operations(const struct nfp_cpp_operations *ops,
+ struct device *parent, void *priv)
+{
+ const u32 arm = NFP_CPP_ID(NFP_CPP_TARGET_ARM, NFP_CPP_ACTION_RW, 0);
+ struct nfp_cpp *cpp;
+ u32 mask[2];
+ u32 xpbaddr;
+ size_t tgt;
+ int err;
+
+ cpp = kzalloc(sizeof(*cpp), GFP_KERNEL);
+ if (!cpp) {
+ err = -ENOMEM;
+ goto err_malloc;
+ }
+
+ cpp->op = ops;
+ cpp->priv = priv;
+ cpp->interface = ops->get_interface(parent);
+ if (ops->read_serial)
+ ops->read_serial(parent, cpp->serial);
+ rwlock_init(&cpp->resource_lock);
+ init_waitqueue_head(&cpp->waitq);
+ lockdep_set_class(&cpp->resource_lock, &nfp_cpp_resource_lock_key);
+ INIT_LIST_HEAD(&cpp->mutex_cache);
+ INIT_LIST_HEAD(&cpp->resource_list);
+ INIT_LIST_HEAD(&cpp->area_cache_list);
+ mutex_init(&cpp->area_cache_mutex);
+ cpp->dev.init_name = "cpp";
+ cpp->dev.parent = parent;
+ cpp->dev.release = nfp_cpp_dev_release;
+ err = device_register(&cpp->dev);
+ if (err < 0) {
+ put_device(&cpp->dev);
+ goto err_dev;
+ }
+
+ dev_set_drvdata(&cpp->dev, cpp);
+
+ /* NOTE: cpp_lock is NOT locked for op->init,
+ * since it may call NFP CPP API operations
+ */
+ if (cpp->op->init) {
+ err = cpp->op->init(cpp);
+ if (err < 0) {
+ dev_err(parent,
+ "NFP interface initialization failed\n");
+ goto err_out;
+ }
+ }
+
+ err = nfp_cpp_model_autodetect(cpp, &cpp->model);
+ if (err < 0) {
+ dev_err(parent, "NFP model detection failed\n");
+ goto err_out;
+ }
+
+ for (tgt = 0; tgt < ARRAY_SIZE(cpp->imb_cat_table); tgt++) {
+ /* Hardcoded XPB IMB Base, island 0 */
+ xpbaddr = 0x000a0000 + (tgt * 4);
+ err = nfp_xpb_readl(cpp, xpbaddr,
+ &cpp->imb_cat_table[tgt]);
+ if (err < 0) {
+ dev_err(parent,
+ "Can't read CPP mapping from device\n");
+ goto err_out;
+ }
+ }
+
+ nfp_cpp_readl(cpp, arm, NFP_ARM_GCSR + NFP_ARM_GCSR_SOFTMODEL2,
+ &mask[0]);
+ nfp_cpp_readl(cpp, arm, NFP_ARM_GCSR + NFP_ARM_GCSR_SOFTMODEL3,
+ &mask[1]);
+
+ dev_info(cpp->dev.parent, "Model: 0x%08x, SN: %pM, Ifc: 0x%04x\n",
+ nfp_cpp_model(cpp), cpp->serial, nfp_cpp_interface(cpp));
+
+ return cpp;
+
+err_out:
+ device_unregister(&cpp->dev);
+err_dev:
+ kfree(cpp);
+err_malloc:
+ return ERR_PTR(err);
+}
+
+/**
+ * nfp_cpp_priv() - Get the operations private data of a CPP handle
+ * @cpp: CPP handle
+ *
+ * Return: Private data for the NFP CPP handle
+ */
+void *nfp_cpp_priv(struct nfp_cpp *cpp)
+{
+ return cpp->priv;
+}
+
+/**
+ * nfp_cpp_device() - Get the Linux device handle of a CPP handle
+ * @cpp: CPP handle
+ *
+ * Return: Device for the NFP CPP bus
+ */
+struct device *nfp_cpp_device(struct nfp_cpp *cpp)
+{
+ return &cpp->dev;
+}
+
+#define NFP_EXPL_OP(func, expl, args...) \
+ ({ \
+ struct nfp_cpp *cpp = nfp_cpp_explicit_cpp(expl); \
+ int err = -ENODEV; \
+ \
+ if (cpp->op->func) \
+ err = cpp->op->func(expl, ##args); \
+ err; \
+ })
+
+#define NFP_EXPL_OP_NR(func, expl, args...) \
+ ({ \
+ struct nfp_cpp *cpp = nfp_cpp_explicit_cpp(expl); \
+ \
+ if (cpp->op->func) \
+ cpp->op->func(expl, ##args); \
+ \
+ })
+
+/**
+ * nfp_cpp_explicit_acquire() - Acquire explicit access handle
+ * @cpp: NFP CPP handle
+ *
+ * The 'data_ref' and 'signal_ref' values are useful when
+ * constructing the NFP_EXPL_CSR1 and NFP_EXPL_POST values.
+ *
+ * Return: NFP CPP explicit handle
+ */
+struct nfp_cpp_explicit *nfp_cpp_explicit_acquire(struct nfp_cpp *cpp)
+{
+ struct nfp_cpp_explicit *expl;
+ int err;
+
+ expl = kzalloc(sizeof(*expl) + cpp->op->explicit_priv_size, GFP_KERNEL);
+ if (!expl)
+ return NULL;
+
+ expl->cpp = cpp;
+ err = NFP_EXPL_OP(explicit_acquire, expl);
+ if (err < 0) {
+ kfree(expl);
+ return NULL;
+ }
+
+ return expl;
+}
+
+/**
+ * nfp_cpp_explicit_set_target() - Set target fields for explicit
+ * @expl: Explicit handle
+ * @cpp_id: CPP ID field
+ * @len: CPP Length field
+ * @mask: CPP Mask field
+ *
+ * Return: 0, or -ERRNO
+ */
+int nfp_cpp_explicit_set_target(struct nfp_cpp_explicit *expl,
+ u32 cpp_id, u8 len, u8 mask)
+{
+ expl->cmd.cpp_id = cpp_id;
+ expl->cmd.len = len;
+ expl->cmd.byte_mask = mask;
+
+ return 0;
+}
+
+/**
+ * nfp_cpp_explicit_set_data() - Set data fields for explicit
+ * @expl: Explicit handle
+ * @data_master: CPP Data Master field
+ * @data_ref: CPP Data Ref field
+ *
+ * Return: 0, or -ERRNO
+ */
+int nfp_cpp_explicit_set_data(struct nfp_cpp_explicit *expl,
+ u8 data_master, u16 data_ref)
+{
+ expl->cmd.data_master = data_master;
+ expl->cmd.data_ref = data_ref;
+
+ return 0;
+}
+
+/**
+ * nfp_cpp_explicit_set_signal() - Set signal fields for explicit
+ * @expl: Explicit handle
+ * @signal_master: CPP Signal Master field
+ * @signal_ref: CPP Signal Ref field
+ *
+ * Return: 0, or -ERRNO
+ */
+int nfp_cpp_explicit_set_signal(struct nfp_cpp_explicit *expl,
+ u8 signal_master, u8 signal_ref)
+{
+ expl->cmd.signal_master = signal_master;
+ expl->cmd.signal_ref = signal_ref;
+
+ return 0;
+}
+
+/**
+ * nfp_cpp_explicit_set_posted() - Set completion fields for explicit
+ * @expl: Explicit handle
+ * @posted: True for signaled completion, false otherwise
+ * @siga: CPP Signal A field
+ * @siga_mode: CPP Signal A Mode field
+ * @sigb: CPP Signal B field
+ * @sigb_mode: CPP Signal B Mode field
+ *
+ * Return: 0, or -ERRNO
+ */
+int nfp_cpp_explicit_set_posted(struct nfp_cpp_explicit *expl, int posted,
+ u8 siga,
+ enum nfp_cpp_explicit_signal_mode siga_mode,
+ u8 sigb,
+ enum nfp_cpp_explicit_signal_mode sigb_mode)
+{
+ expl->cmd.posted = posted;
+ expl->cmd.siga = siga;
+ expl->cmd.sigb = sigb;
+ expl->cmd.siga_mode = siga_mode;
+ expl->cmd.sigb_mode = sigb_mode;
+
+ return 0;
+}
+
+/**
+ * nfp_cpp_explicit_put() - Set up the write (pull) data for a explicit access
+ * @expl: NFP CPP Explicit handle
+ * @buff: Data to have the target pull in the transaction
+ * @len: Length of data, in bytes
+ *
+ * The 'len' parameter must be less than or equal to 128 bytes.
+ *
+ * If this function is called before the configuration
+ * registers are set, it will return -EINVAL.
+ *
+ * Return: 0, or -ERRNO
+ */
+int nfp_cpp_explicit_put(struct nfp_cpp_explicit *expl,
+ const void *buff, size_t len)
+{
+ return NFP_EXPL_OP(explicit_put, expl, buff, len);
+}
+
+/**
+ * nfp_cpp_explicit_do() - Execute a transaction, and wait for it to complete
+ * @expl: NFP CPP Explicit handle
+ * @address: Address to send in the explicit transaction
+ *
+ * If this function is called before the configuration
+ * registers are set, it will return -1, with an errno of EINVAL.
+ *
+ * Return: 0, or -ERRNO
+ */
+int nfp_cpp_explicit_do(struct nfp_cpp_explicit *expl, u64 address)
+{
+ return NFP_EXPL_OP(explicit_do, expl, &expl->cmd, address);
+}
+
+/**
+ * nfp_cpp_explicit_get() - Get the 'push' (read) data from a explicit access
+ * @expl: NFP CPP Explicit handle
+ * @buff: Data that the target pushed in the transaction
+ * @len: Length of data, in bytes
+ *
+ * The 'len' parameter must be less than or equal to 128 bytes.
+ *
+ * If this function is called before all three configuration
+ * registers are set, it will return -1, with an errno of EINVAL.
+ *
+ * If this function is called before nfp_cpp_explicit_do()
+ * has completed, it will return -1, with an errno of EBUSY.
+ *
+ * Return: 0, or -ERRNO
+ */
+int nfp_cpp_explicit_get(struct nfp_cpp_explicit *expl, void *buff, size_t len)
+{
+ return NFP_EXPL_OP(explicit_get, expl, buff, len);
+}
+
+/**
+ * nfp_cpp_explicit_release() - Release explicit access handle
+ * @expl: NFP CPP Explicit handle
+ *
+ */
+void nfp_cpp_explicit_release(struct nfp_cpp_explicit *expl)
+{
+ NFP_EXPL_OP_NR(explicit_release, expl);
+ kfree(expl);
+}
+
+/**
+ * nfp_cpp_explicit_cpp() - return CPP handle for CPP explicit
+ * @cpp_explicit: CPP explicit handle
+ *
+ * Return: NFP CPP handle of the explicit
+ */
+struct nfp_cpp *nfp_cpp_explicit_cpp(struct nfp_cpp_explicit *cpp_explicit)
+{
+ return cpp_explicit->cpp;
+}
+
+/**
+ * nfp_cpp_explicit_priv() - return private struct for CPP explicit
+ * @cpp_explicit: CPP explicit handle
+ *
+ * Return: private data of the explicit, or NULL
+ */
+void *nfp_cpp_explicit_priv(struct nfp_cpp_explicit *cpp_explicit)
+{
+ return &cpp_explicit[1];
+}
+
+/* THIS FUNCTION IS NOT EXPORTED */
+static u32 nfp_mutex_locked(u16 interface)
+{
+ return (u32)interface << 16 | 0x000f;
+}
+
+static u32 nfp_mutex_unlocked(u16 interface)
+{
+ return (u32)interface << 16 | 0x0000;
+}
+
+static bool nfp_mutex_is_locked(u32 val)
+{
+ return (val & 0xffff) == 0x000f;
+}
+
+static bool nfp_mutex_is_unlocked(u32 val)
+{
+ return (val & 0xffff) == 0000;
+}
+
+/* If you need more than 65536 recursive locks, please rethink your code. */
+#define MUTEX_DEPTH_MAX 0xffff
+
+static int
+nfp_cpp_mutex_validate(u16 interface, int *target, unsigned long long address)
+{
+ /* Not permitted on invalid interfaces */
+ if (NFP_CPP_INTERFACE_TYPE_of(interface) ==
+ NFP_CPP_INTERFACE_TYPE_INVALID)
+ return -EINVAL;
+
+ /* Address must be 64-bit aligned */
+ if (address & 7)
+ return -EINVAL;
+
+ if (*target != NFP_CPP_TARGET_MU)
+ return -EINVAL;
+
+ return 0;
+}
+
+/**
+ * nfp_cpp_mutex_init() - Initialize a mutex location
+ * @cpp: NFP CPP handle
+ * @target: NFP CPP target ID (ie NFP_CPP_TARGET_CLS or NFP_CPP_TARGET_MU)
+ * @address: Offset into the address space of the NFP CPP target ID
+ * @key: Unique 32-bit value for this mutex
+ *
+ * The CPP target:address must point to a 64-bit aligned location, and
+ * will initialize 64 bits of data at the location.
+ *
+ * This creates the initial mutex state, as locked by this
+ * nfp_cpp_interface().
+ *
+ * This function should only be called when setting up
+ * the initial lock state upon boot-up of the system.
+ *
+ * Return: 0 on success, or -errno on failure
+ */
+int nfp_cpp_mutex_init(struct nfp_cpp *cpp,
+ int target, unsigned long long address, u32 key)
+{
+ const u32 muw = NFP_CPP_ID(target, 4, 0); /* atomic_write */
+ u16 interface = nfp_cpp_interface(cpp);
+ int err;
+
+ err = nfp_cpp_mutex_validate(interface, &target, address);
+ if (err)
+ return err;
+
+ err = nfp_cpp_writel(cpp, muw, address + 4, key);
+ if (err)
+ return err;
+
+ err = nfp_cpp_writel(cpp, muw, address, nfp_mutex_locked(interface));
+ if (err)
+ return err;
+
+ return 0;
+}
+
+/**
+ * nfp_cpp_mutex_alloc() - Create a mutex handle
+ * @cpp: NFP CPP handle
+ * @target: NFP CPP target ID (ie NFP_CPP_TARGET_CLS or NFP_CPP_TARGET_MU)
+ * @address: Offset into the address space of the NFP CPP target ID
+ * @key: 32-bit unique key (must match the key at this location)
+ *
+ * The CPP target:address must point to a 64-bit aligned location, and
+ * reserve 64 bits of data at the location for use by the handle.
+ *
+ * Only target/address pairs that point to entities that support the
+ * MU Atomic Engine's CmpAndSwap32 command are supported.
+ *
+ * Return: A non-NULL struct nfp_cpp_mutex * on success, NULL on failure.
+ */
+struct nfp_cpp_mutex *nfp_cpp_mutex_alloc(struct nfp_cpp *cpp, int target,
+ unsigned long long address, u32 key)
+{
+ const u32 mur = NFP_CPP_ID(target, 3, 0); /* atomic_read */
+ u16 interface = nfp_cpp_interface(cpp);
+ struct nfp_cpp_mutex *mutex;
+ int err;
+ u32 tmp;
+
+ err = nfp_cpp_mutex_validate(interface, &target, address);
+ if (err)
+ return NULL;
+
+ /* Look for mutex on cache list */
+ list_for_each_entry(mutex, &cpp->mutex_cache, list) {
+ if (mutex->target == target && mutex->address == address) {
+ mutex->usage++;
+ return mutex;
+ }
+ }
+
+ err = nfp_cpp_readl(cpp, mur, address + 4, &tmp);
+ if (err < 0)
+ return NULL;
+
+ if (tmp != key)
+ return NULL;
+
+ mutex = kzalloc(sizeof(*mutex), GFP_KERNEL);
+ if (!mutex)
+ return NULL;
+
+ mutex->cpp = cpp;
+ mutex->target = target;
+ mutex->address = address;
+ mutex->key = key;
+ mutex->depth = 0;
+ mutex->usage = 1;
+
+ /* Add mutex to cache list */
+ list_add(&mutex->list, &cpp->mutex_cache);
+
+ return mutex;
+}
+
+/**
+ * nfp_cpp_mutex_free() - Free a mutex handle - does not alter the lock state
+ * @mutex: NFP CPP Mutex handle
+ */
+void nfp_cpp_mutex_free(struct nfp_cpp_mutex *mutex)
+{
+ if (--mutex->usage)
+ return;
+
+ /* Remove mutex from cache */
+ list_del(&mutex->list);
+ kfree(mutex);
+}
+
+/**
+ * nfp_cpp_mutex_lock() - Lock a mutex handle, using the NFP MU Atomic Engine
+ * @mutex: NFP CPP Mutex handle
+ *
+ * Return: 0 on success, or -errno on failure
+ */
+int nfp_cpp_mutex_lock(struct nfp_cpp_mutex *mutex)
+{
+ unsigned long warn_at = jiffies + 15 * HZ;
+ unsigned int timeout_ms = 1;
+ int err;
+
+ /* We can't use a waitqueue here, because the unlocker
+ * might be on a separate CPU.
+ *
+ * So just wait for now.
+ */
+ for (;;) {
+ err = nfp_cpp_mutex_trylock(mutex);
+ if (err != -EBUSY)
+ break;
+
+ err = msleep_interruptible(timeout_ms);
+ if (err != 0)
+ return -ERESTARTSYS;
+
+ if (time_is_before_eq_jiffies(warn_at)) {
+ warn_at = jiffies + 60 * HZ;
+ dev_warn(mutex->cpp->dev.parent,
+ "Warning: waiting for NFP mutex [usage:%hd depth:%hd target:%d addr:%llx key:%08x]\n",
+ mutex->usage, mutex->depth,
+ mutex->target, mutex->address, mutex->key);
+ }
+ }
+
+ return err;
+}
+
+/**
+ * nfp_cpp_mutex_unlock() - Unlock a mutex handle, using the MU Atomic Engine
+ * @mutex: NFP CPP Mutex handle
+ *
+ * Return: 0 on success, or -errno on failure
+ */
+int nfp_cpp_mutex_unlock(struct nfp_cpp_mutex *mutex)
+{
+ const u32 muw = NFP_CPP_ID(mutex->target, 4, 0); /* atomic_write */
+ const u32 mur = NFP_CPP_ID(mutex->target, 3, 0); /* atomic_read */
+ struct nfp_cpp *cpp = mutex->cpp;
+ u32 key, value;
+ u16 interface;
+ int err;
+
+ interface = nfp_cpp_interface(cpp);
+
+ if (mutex->depth > 1) {
+ mutex->depth--;
+ return 0;
+ }
+
+ err = nfp_cpp_readl(mutex->cpp, mur, mutex->address + 4, &key);
+ if (err < 0)
+ return err;
+
+ if (key != mutex->key)
+ return -EPERM;
+
+ err = nfp_cpp_readl(mutex->cpp, mur, mutex->address, &value);
+ if (err < 0)
+ return err;
+
+ if (value != nfp_mutex_locked(interface))
+ return -EACCES;
+
+ err = nfp_cpp_writel(cpp, muw, mutex->address,
+ nfp_mutex_unlocked(interface));
+ if (err < 0)
+ return err;
+
+ mutex->depth = 0;
+ return 0;
+}
+
+/**
+ * nfp_cpp_mutex_trylock() - Attempt to lock a mutex handle
+ * @mutex: NFP CPP Mutex handle
+ *
+ * Return: 0 if the lock succeeded, -errno on failure
+ */
+int nfp_cpp_mutex_trylock(struct nfp_cpp_mutex *mutex)
+{
+ const u32 muw = NFP_CPP_ID(mutex->target, 4, 0); /* atomic_write */
+ const u32 mus = NFP_CPP_ID(mutex->target, 5, 3); /* test_set_imm */
+ const u32 mur = NFP_CPP_ID(mutex->target, 3, 0); /* atomic_read */
+ struct nfp_cpp *cpp = mutex->cpp;
+ u32 key, value, tmp;
+ int err;
+
+ if (mutex->depth > 0) {
+ if (mutex->depth == MUTEX_DEPTH_MAX)
+ return -E2BIG;
+ mutex->depth++;
+ return 0;
+ }
+
+ /* Verify that the lock marker is not damaged */
+ err = nfp_cpp_readl(cpp, mur, mutex->address + 4, &key);
+ if (err < 0)
+ return err;
+
+ if (key != mutex->key)
+ return -EPERM;
+
+ /* Compare against the unlocked state, and if true,
+ * write the interface id into the top 16 bits, and
+ * mark as locked.
+ */
+ value = nfp_mutex_locked(nfp_cpp_interface(cpp));
+
+ /* We use test_set_imm here, as it implies a read
+ * of the current state, and sets the bits in the
+ * bytemask of the command to 1s. Since the mutex
+ * is guaranteed to be 64-bit aligned, the bytemask
+ * of this 32-bit command is ensured to be 8'b00001111,
+ * which implies that the lower 4 bits will be set to
+ * ones regardless of the initial state.
+ *
+ * Since this is a 'Readback' operation, with no Pull
+ * data, we can treat this as a normal Push (read)
+ * atomic, which returns the original value.
+ */
+ err = nfp_cpp_readl(cpp, mus, mutex->address, &tmp);
+ if (err < 0)
+ return err;
+
+ /* Was it unlocked? */
+ if (nfp_mutex_is_unlocked(tmp)) {
+ /* The read value can only be 0x....0000 in the unlocked state.
+ * If there was another contending for this lock, then
+ * the lock state would be 0x....000f
+ */
+
+ /* Write our owner ID into the lock
+ * While not strictly necessary, this helps with
+ * debug and bookkeeping.
+ */
+ err = nfp_cpp_writel(cpp, muw, mutex->address, value);
+ if (err < 0)
+ return err;
+
+ mutex->depth = 1;
+ return 0;
+ }
+
+ /* Already locked by us? Success! */
+ if (tmp == value) {
+ mutex->depth = 1;
+ return 0;
+ }
+
+ return nfp_mutex_is_locked(tmp) ? -EBUSY : -EINVAL;
+}
--- /dev/null
+/*
+ * Copyright (C) 2015-2017 Netronome Systems, Inc.
+ *
+ * This software is dual licensed under the GNU General License Version 2,
+ * June 1991 as shown in the file COPYING in the top-level directory of this
+ * source tree or the BSD 2-Clause License provided below. You have the
+ * option to license this software under the complete terms of either license.
+ *
+ * The BSD 2-Clause License:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+/*
+ * nfp_cpplib.c
+ * Library of functions to access the NFP's CPP bus
+ * Authors: Jakub Kicinski <jakub.kicinski@netronome.com>
+ * Jason McMullan <jason.mcmullan@netronome.com>
+ * Rolf Neugebauer <rolf.neugebauer@netronome.com>
+ */
+
+#include <asm/unaligned.h>
+#include <linux/bitfield.h>
+#include <linux/delay.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/sched.h>
+
+#include "nfp_cpp.h"
+#include "nfp6000/nfp6000.h"
+#include "nfp6000/nfp_xpb.h"
+
+/* NFP6000 PL */
+#define NFP_PL_DEVICE_ID 0x00000004
+#define NFP_PL_DEVICE_ID_MASK GENMASK(7, 0)
+
+#define NFP6000_ARM_GCSR_SOFTMODEL0 0x00400144
+
+/**
+ * nfp_cpp_readl() - Read a u32 word from a CPP location
+ * @cpp: CPP device handle
+ * @cpp_id: CPP ID for operation
+ * @address: Address for operation
+ * @value: Pointer to read buffer
+ *
+ * Return: length of the io, or -ERRNO
+ */
+int nfp_cpp_readl(struct nfp_cpp *cpp, u32 cpp_id,
+ unsigned long long address, u32 *value)
+{
+ u8 tmp[4];
+ int err;
+
+ err = nfp_cpp_read(cpp, cpp_id, address, tmp, sizeof(tmp));
+ *value = get_unaligned_le32(tmp);
+
+ return err;
+}
+
+/**
+ * nfp_cpp_writel() - Write a u32 word to a CPP location
+ * @cpp: CPP device handle
+ * @cpp_id: CPP ID for operation
+ * @address: Address for operation
+ * @value: Value to write
+ *
+ * Return: length of the io, or -ERRNO
+ */
+int nfp_cpp_writel(struct nfp_cpp *cpp, u32 cpp_id,
+ unsigned long long address, u32 value)
+{
+ u8 tmp[4];
+
+ put_unaligned_le32(value, tmp);
+ return nfp_cpp_write(cpp, cpp_id, address, tmp, sizeof(tmp));
+}
+
+/**
+ * nfp_cpp_readq() - Read a u64 word from a CPP location
+ * @cpp: CPP device handle
+ * @cpp_id: CPP ID for operation
+ * @address: Address for operation
+ * @value: Pointer to read buffer
+ *
+ * Return: length of the io, or -ERRNO
+ */
+int nfp_cpp_readq(struct nfp_cpp *cpp, u32 cpp_id,
+ unsigned long long address, u64 *value)
+{
+ u8 tmp[8];
+ int err;
+
+ err = nfp_cpp_read(cpp, cpp_id, address, tmp, sizeof(tmp));
+ *value = get_unaligned_le64(tmp);
+
+ return err;
+}
+
+/**
+ * nfp_cpp_writeq() - Write a u64 word to a CPP location
+ * @cpp: CPP device handle
+ * @cpp_id: CPP ID for operation
+ * @address: Address for operation
+ * @value: Value to write
+ *
+ * Return: length of the io, or -ERRNO
+ */
+int nfp_cpp_writeq(struct nfp_cpp *cpp, u32 cpp_id,
+ unsigned long long address, u64 value)
+{
+ u8 tmp[8];
+
+ put_unaligned_le64(value, tmp);
+ return nfp_cpp_write(cpp, cpp_id, address, tmp, sizeof(tmp));
+}
+
+/* NOTE: This code should not use nfp_xpb_* functions,
+ * as those are model-specific
+ */
+int nfp_cpp_model_autodetect(struct nfp_cpp *cpp, u32 *model)
+{
+ const u32 arm_id = NFP_CPP_ID(NFP_CPP_TARGET_ARM, 0, 0);
+ u32 reg;
+ int err;
+
+ err = nfp_cpp_readl(cpp, arm_id, NFP6000_ARM_GCSR_SOFTMODEL0, model);
+ if (err < 0)
+ return err;
+
+ /* The PL's PluDeviceID revision code is authoratative */
+ *model &= ~0xff;
+ err = nfp_xpb_readl(cpp, NFP_XPB_DEVICE(1, 1, 16) + NFP_PL_DEVICE_ID,
+ ®);
+ if (err < 0)
+ return err;
+
+ *model |= (NFP_PL_DEVICE_ID_MASK & reg) - 0x10;
+
+ return 0;
+}
+
+static u8 nfp_bytemask(int width, u64 addr)
+{
+ if (width == 8)
+ return 0xff;
+ else if (width == 4)
+ return 0x0f << (addr & 4);
+ else if (width == 2)
+ return 0x03 << (addr & 6);
+ else if (width == 1)
+ return 0x01 << (addr & 7);
+ else
+ return 0;
+}
+
+int nfp_cpp_explicit_read(struct nfp_cpp *cpp, u32 cpp_id,
+ u64 addr, void *buff, size_t len, int width_read)
+{
+ struct nfp_cpp_explicit *expl;
+ char *tmp = buff;
+ int err, i, incr;
+ u8 byte_mask;
+
+ if (len & (width_read - 1))
+ return -EINVAL;
+
+ expl = nfp_cpp_explicit_acquire(cpp);
+ if (!expl)
+ return -EBUSY;
+
+ incr = min_t(int, 16 * width_read, 128);
+ incr = min_t(int, incr, len);
+
+ /* Translate a NFP_CPP_ACTION_RW to action 0 */
+ if (NFP_CPP_ID_ACTION_of(cpp_id) == NFP_CPP_ACTION_RW)
+ cpp_id = NFP_CPP_ID(NFP_CPP_ID_TARGET_of(cpp_id), 0,
+ NFP_CPP_ID_TOKEN_of(cpp_id));
+
+ byte_mask = nfp_bytemask(width_read, addr);
+
+ nfp_cpp_explicit_set_target(expl, cpp_id,
+ incr / width_read - 1, byte_mask);
+ nfp_cpp_explicit_set_posted(expl, 1, 0, NFP_SIGNAL_PUSH,
+ 0, NFP_SIGNAL_NONE);
+
+ for (i = 0; i < len; i += incr, addr += incr, tmp += incr) {
+ if (i + incr > len) {
+ incr = len - i;
+ nfp_cpp_explicit_set_target(expl, cpp_id,
+ incr / width_read - 1,
+ 0xff);
+ }
+
+ err = nfp_cpp_explicit_do(expl, addr);
+ if (err < 0)
+ goto exit_release;
+
+ err = nfp_cpp_explicit_get(expl, tmp, incr);
+ if (err < 0)
+ goto exit_release;
+ }
+ err = len;
+exit_release:
+ nfp_cpp_explicit_release(expl);
+
+ return err;
+}
+
+int nfp_cpp_explicit_write(struct nfp_cpp *cpp, u32 cpp_id, u64 addr,
+ const void *buff, size_t len, int width_write)
+{
+ struct nfp_cpp_explicit *expl;
+ const char *tmp = buff;
+ int err, i, incr;
+ u8 byte_mask;
+
+ if (len & (width_write - 1))
+ return -EINVAL;
+
+ expl = nfp_cpp_explicit_acquire(cpp);
+ if (!expl)
+ return -EBUSY;
+
+ incr = min_t(int, 16 * width_write, 128);
+ incr = min_t(int, incr, len);
+
+ /* Translate a NFP_CPP_ACTION_RW to action 1 */
+ if (NFP_CPP_ID_ACTION_of(cpp_id) == NFP_CPP_ACTION_RW)
+ cpp_id = NFP_CPP_ID(NFP_CPP_ID_TARGET_of(cpp_id), 1,
+ NFP_CPP_ID_TOKEN_of(cpp_id));
+
+ byte_mask = nfp_bytemask(width_write, addr);
+
+ nfp_cpp_explicit_set_target(expl, cpp_id,
+ incr / width_write - 1, byte_mask);
+ nfp_cpp_explicit_set_posted(expl, 1, 0, NFP_SIGNAL_PULL,
+ 0, NFP_SIGNAL_NONE);
+
+ for (i = 0; i < len; i += incr, addr += incr, tmp += incr) {
+ if (i + incr > len) {
+ incr = len - i;
+ nfp_cpp_explicit_set_target(expl, cpp_id,
+ incr / width_write - 1,
+ 0xff);
+ }
+
+ err = nfp_cpp_explicit_put(expl, tmp, incr);
+ if (err < 0)
+ goto exit_release;
+
+ err = nfp_cpp_explicit_do(expl, addr);
+ if (err < 0)
+ goto exit_release;
+ }
+ err = len;
+exit_release:
+ nfp_cpp_explicit_release(expl);
+
+ return err;
+}
--- /dev/null
+/*
+ * Copyright (C) 2015-2017 Netronome Systems, Inc.
+ *
+ * This software is dual licensed under the GNU General License Version 2,
+ * June 1991 as shown in the file COPYING in the top-level directory of this
+ * source tree or the BSD 2-Clause License provided below. You have the
+ * option to license this software under the complete terms of either license.
+ *
+ * The BSD 2-Clause License:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+/*
+ * nfp_target.c
+ * CPP Access Width Decoder
+ * Authors: Jakub Kicinski <jakub.kicinski@netronome.com>
+ * Jason McMullan <jason.mcmullan@netronome.com>
+ * Francois H. Theron <francois.theron@netronome.com>
+ */
+
+#include <linux/bitops.h>
+
+#include "nfp_cpp.h"
+
+#include "nfp6000/nfp6000.h"
+
+#define P32 1
+#define P64 2
+
+/* This structure ONLY includes items that can be done with a read or write of
+ * 32-bit or 64-bit words. All others are not listed.
+ */
+
+#define AT(_action, _token, _pull, _push) \
+ case NFP_CPP_ID(0, (_action), (_token)): \
+ return PUSHPULL((_pull), (_push))
+
+static int target_rw(u32 cpp_id, int pp, int start, int len)
+{
+ switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
+ AT(0, 0, 0, pp);
+ AT(1, 0, pp, 0);
+ AT(NFP_CPP_ACTION_RW, 0, pp, pp);
+ default:
+ return -EINVAL;
+ }
+}
+
+static int nfp6000_nbi_dma(u32 cpp_id)
+{
+ switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
+ AT(0, 0, 0, P64); /* ReadNbiDma */
+ AT(1, 0, P64, 0); /* WriteNbiDma */
+ AT(NFP_CPP_ACTION_RW, 0, P64, P64);
+ default:
+ return -EINVAL;
+ }
+}
+
+static int nfp6000_nbi_stats(u32 cpp_id)
+{
+ switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
+ AT(0, 0, 0, P32); /* ReadNbiStats */
+ AT(1, 0, P32, 0); /* WriteNbiStats */
+ AT(NFP_CPP_ACTION_RW, 0, P32, P32);
+ default:
+ return -EINVAL;
+ }
+}
+
+static int nfp6000_nbi_tm(u32 cpp_id)
+{
+ switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
+ AT(0, 0, 0, P64); /* ReadNbiTM */
+ AT(1, 0, P64, 0); /* WriteNbiTM */
+ AT(NFP_CPP_ACTION_RW, 0, P64, P64);
+ default:
+ return -EINVAL;
+ }
+}
+
+static int nfp6000_nbi_ppc(u32 cpp_id)
+{
+ switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
+ AT(0, 0, 0, P64); /* ReadNbiPreclassifier */
+ AT(1, 0, P64, 0); /* WriteNbiPreclassifier */
+ AT(NFP_CPP_ACTION_RW, 0, P64, P64);
+ default:
+ return -EINVAL;
+ }
+}
+
+static int nfp6000_nbi(u32 cpp_id, u64 address)
+{
+ u64 rel_addr = address & 0x3fFFFF;
+
+ if (rel_addr < (1 << 20))
+ return nfp6000_nbi_dma(cpp_id);
+ if (rel_addr < (2 << 20))
+ return nfp6000_nbi_stats(cpp_id);
+ if (rel_addr < (3 << 20))
+ return nfp6000_nbi_tm(cpp_id);
+ return nfp6000_nbi_ppc(cpp_id);
+}
+
+/* This structure ONLY includes items that can be done with a read or write of
+ * 32-bit or 64-bit words. All others are not listed.
+ */
+static int nfp6000_mu_common(u32 cpp_id)
+{
+ switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
+ AT(NFP_CPP_ACTION_RW, 0, P64, P64); /* read_be/write_be */
+ AT(NFP_CPP_ACTION_RW, 1, P64, P64); /* read_le/write_le */
+ AT(NFP_CPP_ACTION_RW, 2, P64, P64); /* read_swap_be/write_swap_be */
+ AT(NFP_CPP_ACTION_RW, 3, P64, P64); /* read_swap_le/write_swap_le */
+ AT(0, 0, 0, P64); /* read_be */
+ AT(0, 1, 0, P64); /* read_le */
+ AT(0, 2, 0, P64); /* read_swap_be */
+ AT(0, 3, 0, P64); /* read_swap_le */
+ AT(1, 0, P64, 0); /* write_be */
+ AT(1, 1, P64, 0); /* write_le */
+ AT(1, 2, P64, 0); /* write_swap_be */
+ AT(1, 3, P64, 0); /* write_swap_le */
+ AT(3, 0, 0, P32); /* atomic_read */
+ AT(3, 2, P32, 0); /* mask_compare_write */
+ AT(4, 0, P32, 0); /* atomic_write */
+ AT(4, 2, 0, 0); /* atomic_write_imm */
+ AT(4, 3, 0, P32); /* swap_imm */
+ AT(5, 0, P32, 0); /* set */
+ AT(5, 3, 0, P32); /* test_set_imm */
+ AT(6, 0, P32, 0); /* clr */
+ AT(6, 3, 0, P32); /* test_clr_imm */
+ AT(7, 0, P32, 0); /* add */
+ AT(7, 3, 0, P32); /* test_add_imm */
+ AT(8, 0, P32, 0); /* addsat */
+ AT(8, 3, 0, P32); /* test_subsat_imm */
+ AT(9, 0, P32, 0); /* sub */
+ AT(9, 3, 0, P32); /* test_sub_imm */
+ AT(10, 0, P32, 0); /* subsat */
+ AT(10, 3, 0, P32); /* test_subsat_imm */
+ AT(13, 0, 0, P32); /* microq128_get */
+ AT(13, 1, 0, P32); /* microq128_pop */
+ AT(13, 2, P32, 0); /* microq128_put */
+ AT(15, 0, P32, 0); /* xor */
+ AT(15, 3, 0, P32); /* test_xor_imm */
+ AT(28, 0, 0, P32); /* read32_be */
+ AT(28, 1, 0, P32); /* read32_le */
+ AT(28, 2, 0, P32); /* read32_swap_be */
+ AT(28, 3, 0, P32); /* read32_swap_le */
+ AT(31, 0, P32, 0); /* write32_be */
+ AT(31, 1, P32, 0); /* write32_le */
+ AT(31, 2, P32, 0); /* write32_swap_be */
+ AT(31, 3, P32, 0); /* write32_swap_le */
+ default:
+ return -EINVAL;
+ }
+}
+
+static int nfp6000_mu_ctm(u32 cpp_id)
+{
+ switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
+ AT(16, 1, 0, P32); /* packet_read_packet_status */
+ AT(17, 1, 0, P32); /* packet_credit_get */
+ AT(17, 3, 0, P64); /* packet_add_thread */
+ AT(18, 2, 0, P64); /* packet_free_and_return_pointer */
+ AT(18, 3, 0, P64); /* packet_return_pointer */
+ AT(21, 0, 0, P64); /* pe_dma_to_memory_indirect */
+ AT(21, 1, 0, P64); /* pe_dma_to_memory_indirect_swap */
+ AT(21, 2, 0, P64); /* pe_dma_to_memory_indirect_free */
+ AT(21, 3, 0, P64); /* pe_dma_to_memory_indirect_free_swap */
+ default:
+ return nfp6000_mu_common(cpp_id);
+ }
+}
+
+static int nfp6000_mu_emu(u32 cpp_id)
+{
+ switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
+ AT(18, 0, 0, P32); /* read_queue */
+ AT(18, 1, 0, P32); /* read_queue_ring */
+ AT(18, 2, P32, 0); /* write_queue */
+ AT(18, 3, P32, 0); /* write_queue_ring */
+ AT(20, 2, P32, 0); /* journal */
+ AT(21, 0, 0, P32); /* get */
+ AT(21, 1, 0, P32); /* get_eop */
+ AT(21, 2, 0, P32); /* get_freely */
+ AT(22, 0, 0, P32); /* pop */
+ AT(22, 1, 0, P32); /* pop_eop */
+ AT(22, 2, 0, P32); /* pop_freely */
+ default:
+ return nfp6000_mu_common(cpp_id);
+ }
+}
+
+static int nfp6000_mu_imu(u32 cpp_id)
+{
+ return nfp6000_mu_common(cpp_id);
+}
+
+static int nfp6000_mu(u32 cpp_id, u64 address)
+{
+ int pp;
+
+ if (address < 0x2000000000ULL)
+ pp = nfp6000_mu_ctm(cpp_id);
+ else if (address < 0x8000000000ULL)
+ pp = nfp6000_mu_emu(cpp_id);
+ else if (address < 0x9800000000ULL)
+ pp = nfp6000_mu_ctm(cpp_id);
+ else if (address < 0x9C00000000ULL)
+ pp = nfp6000_mu_emu(cpp_id);
+ else if (address < 0xA000000000ULL)
+ pp = nfp6000_mu_imu(cpp_id);
+ else
+ pp = nfp6000_mu_ctm(cpp_id);
+
+ return pp;
+}
+
+static int nfp6000_ila(u32 cpp_id)
+{
+ switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
+ AT(0, 1, 0, P32); /* read_check_error */
+ AT(2, 0, 0, P32); /* read_int */
+ AT(3, 0, P32, 0); /* write_int */
+ default:
+ return target_rw(cpp_id, P32, 48, 4);
+ }
+}
+
+static int nfp6000_pci(u32 cpp_id)
+{
+ switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
+ AT(2, 0, 0, P32);
+ AT(3, 0, P32, 0);
+ default:
+ return target_rw(cpp_id, P32, 4, 4);
+ }
+}
+
+static int nfp6000_crypto(u32 cpp_id)
+{
+ switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
+ AT(2, 0, P64, 0);
+ default:
+ return target_rw(cpp_id, P64, 12, 4);
+ }
+}
+
+static int nfp6000_cap_xpb(u32 cpp_id)
+{
+ switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
+ AT(0, 1, 0, P32); /* RingGet */
+ AT(0, 2, P32, 0); /* Interthread Signal */
+ AT(1, 1, P32, 0); /* RingPut */
+ AT(1, 2, P32, 0); /* CTNNWr */
+ AT(2, 0, 0, P32); /* ReflectRd, signal none */
+ AT(2, 1, 0, P32); /* ReflectRd, signal self */
+ AT(2, 2, 0, P32); /* ReflectRd, signal remote */
+ AT(2, 3, 0, P32); /* ReflectRd, signal both */
+ AT(3, 0, P32, 0); /* ReflectWr, signal none */
+ AT(3, 1, P32, 0); /* ReflectWr, signal self */
+ AT(3, 2, P32, 0); /* ReflectWr, signal remote */
+ AT(3, 3, P32, 0); /* ReflectWr, signal both */
+ AT(NFP_CPP_ACTION_RW, 1, P32, P32);
+ default:
+ return target_rw(cpp_id, P32, 1, 63);
+ }
+}
+
+static int nfp6000_cls(u32 cpp_id)
+{
+ switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
+ AT(0, 3, P32, 0); /* xor */
+ AT(2, 0, P32, 0); /* set */
+ AT(2, 1, P32, 0); /* clr */
+ AT(4, 0, P32, 0); /* add */
+ AT(4, 1, P32, 0); /* add64 */
+ AT(6, 0, P32, 0); /* sub */
+ AT(6, 1, P32, 0); /* sub64 */
+ AT(6, 2, P32, 0); /* subsat */
+ AT(8, 2, P32, 0); /* hash_mask */
+ AT(8, 3, P32, 0); /* hash_clear */
+ AT(9, 0, 0, P32); /* ring_get */
+ AT(9, 1, 0, P32); /* ring_pop */
+ AT(9, 2, 0, P32); /* ring_get_freely */
+ AT(9, 3, 0, P32); /* ring_pop_freely */
+ AT(10, 0, P32, 0); /* ring_put */
+ AT(10, 2, P32, 0); /* ring_journal */
+ AT(14, 0, P32, 0); /* reflect_write_sig_local */
+ AT(15, 1, 0, P32); /* reflect_read_sig_local */
+ AT(17, 2, P32, 0); /* statisic */
+ AT(24, 0, 0, P32); /* ring_read */
+ AT(24, 1, P32, 0); /* ring_write */
+ AT(25, 0, 0, P32); /* ring_workq_add_thread */
+ AT(25, 1, P32, 0); /* ring_workq_add_work */
+ default:
+ return target_rw(cpp_id, P32, 0, 64);
+ }
+}
+
+int nfp_target_pushpull(u32 cpp_id, u64 address)
+{
+ switch (NFP_CPP_ID_TARGET_of(cpp_id)) {
+ case NFP_CPP_TARGET_NBI:
+ return nfp6000_nbi(cpp_id, address);
+ case NFP_CPP_TARGET_QDR:
+ return target_rw(cpp_id, P32, 24, 4);
+ case NFP_CPP_TARGET_ILA:
+ return nfp6000_ila(cpp_id);
+ case NFP_CPP_TARGET_MU:
+ return nfp6000_mu(cpp_id, address);
+ case NFP_CPP_TARGET_PCIE:
+ return nfp6000_pci(cpp_id);
+ case NFP_CPP_TARGET_ARM:
+ if (address < 0x10000)
+ return target_rw(cpp_id, P64, 1, 1);
+ else
+ return target_rw(cpp_id, P32, 1, 1);
+ case NFP_CPP_TARGET_CRYPTO:
+ return nfp6000_crypto(cpp_id);
+ case NFP_CPP_TARGET_CT_XPB:
+ return nfp6000_cap_xpb(cpp_id);
+ case NFP_CPP_TARGET_CLS:
+ return nfp6000_cls(cpp_id);
+ case 0:
+ return target_rw(cpp_id, P32, 4, 4);
+ default:
+ return -EINVAL;
+ }
+}
+
+#undef AT
+#undef P32
+#undef P64
+
+/* All magic NFP-6xxx IMB 'mode' numbers here are from:
+ * Databook (1 August 2013)
+ * - System Overview and Connectivity
+ * -- Internal Connectivity
+ * --- Distributed Switch Fabric - Command Push/Pull (DSF-CPP) Bus
+ * ---- CPP addressing
+ * ----- Table 3.6. CPP Address Translation Mode Commands
+ */
+
+#define _NIC_NFP6000_MU_LOCALITY_DIRECT 2
+
+static int nfp_decode_basic(u64 addr, int *dest_island, int cpp_tgt,
+ int mode, bool addr40, int isld1, int isld0)
+{
+ int iid_lsb, idx_lsb;
+
+ /* This function doesn't handle MU or CTXBP */
+ if (cpp_tgt == NFP_CPP_TARGET_MU || cpp_tgt == NFP_CPP_TARGET_CT_XPB)
+ return -EINVAL;
+
+ switch (mode) {
+ case 0:
+ /* For VQDR, in this mode for 32-bit addressing
+ * it would be islands 0, 16, 32 and 48 depending on channel
+ * and upper address bits.
+ * Since those are not all valid islands, most decode
+ * cases would result in bad island IDs, but we do them
+ * anyway since this is decoding an address that is already
+ * assumed to be used as-is to get to sram.
+ */
+ iid_lsb = addr40 ? 34 : 26;
+ *dest_island = (addr >> iid_lsb) & 0x3F;
+ return 0;
+ case 1:
+ /* For VQDR 32-bit, this would decode as:
+ * Channel 0: island#0
+ * Channel 1: island#0
+ * Channel 2: island#1
+ * Channel 3: island#1
+ * That would be valid as long as both islands
+ * have VQDR. Let's allow this.
+ */
+ idx_lsb = addr40 ? 39 : 31;
+ if (addr & BIT_ULL(idx_lsb))
+ *dest_island = isld1;
+ else
+ *dest_island = isld0;
+
+ return 0;
+ case 2:
+ /* For VQDR 32-bit:
+ * Channel 0: (island#0 | 0)
+ * Channel 1: (island#0 | 1)
+ * Channel 2: (island#1 | 0)
+ * Channel 3: (island#1 | 1)
+ *
+ * Make sure we compare against isldN values
+ * by clearing the LSB.
+ * This is what the silicon does.
+ */
+ isld0 &= ~1;
+ isld1 &= ~1;
+
+ idx_lsb = addr40 ? 39 : 31;
+ iid_lsb = idx_lsb - 1;
+
+ if (addr & BIT_ULL(idx_lsb))
+ *dest_island = isld1 | (int)((addr >> iid_lsb) & 1);
+ else
+ *dest_island = isld0 | (int)((addr >> iid_lsb) & 1);
+
+ return 0;
+ case 3:
+ /* In this mode the data address starts to affect the island ID
+ * so rather not allow it. In some really specific case
+ * one could use this to send the upper half of the
+ * VQDR channel to another MU, but this is getting very
+ * specific.
+ * However, as above for mode 0, this is the decoder
+ * and the caller should validate the resulting IID.
+ * This blindly does what the silicon would do.
+ */
+ isld0 &= ~3;
+ isld1 &= ~3;
+
+ idx_lsb = addr40 ? 39 : 31;
+ iid_lsb = idx_lsb - 2;
+
+ if (addr & BIT_ULL(idx_lsb))
+ *dest_island = isld1 | (int)((addr >> iid_lsb) & 3);
+ else
+ *dest_island = isld0 | (int)((addr >> iid_lsb) & 3);
+
+ return 0;
+ default:
+ return -EINVAL;
+ }
+}
+
+static int nfp_encode_basic_qdr(u64 addr, int dest_island, int cpp_tgt,
+ int mode, bool addr40, int isld1, int isld0)
+{
+ int v, ret;
+
+ /* Full Island ID and channel bits overlap? */
+ ret = nfp_decode_basic(addr, &v, cpp_tgt, mode, addr40, isld1, isld0);
+ if (ret)
+ return ret;
+
+ /* The current address won't go where expected? */
+ if (dest_island != -1 && dest_island != v)
+ return -EINVAL;
+
+ /* If dest_island was -1, we don't care where it goes. */
+ return 0;
+}
+
+/* Try each option, take first one that fits.
+ * Not sure if we would want to do some smarter
+ * searching and prefer 0 or non-0 island IDs.
+ */
+static int nfp_encode_basic_search(u64 *addr, int dest_island, int *isld,
+ int iid_lsb, int idx_lsb, int v_max)
+{
+ int i, v;
+
+ for (i = 0; i < 2; i++)
+ for (v = 0; v < v_max; v++) {
+ if (dest_island != (isld[i] | v))
+ continue;
+
+ *addr &= ~GENMASK_ULL(idx_lsb, iid_lsb);
+ *addr |= ((u64)i << idx_lsb);
+ *addr |= ((u64)v << iid_lsb);
+ return 0;
+ }
+
+ return -ENODEV;
+}
+
+/* For VQDR, we may not modify the Channel bits, which might overlap
+ * with the Index bit. When it does, we need to ensure that isld0 == isld1.
+ */
+static int nfp_encode_basic(u64 *addr, int dest_island, int cpp_tgt,
+ int mode, bool addr40, int isld1, int isld0)
+{
+ int iid_lsb, idx_lsb;
+ int isld[2];
+ u64 v64;
+
+ isld[0] = isld0;
+ isld[1] = isld1;
+
+ /* This function doesn't handle MU or CTXBP */
+ if (cpp_tgt == NFP_CPP_TARGET_MU || cpp_tgt == NFP_CPP_TARGET_CT_XPB)
+ return -EINVAL;
+
+ switch (mode) {
+ case 0:
+ if (cpp_tgt == NFP_CPP_TARGET_QDR && !addr40)
+ /* In this specific mode we'd rather not modify
+ * the address but we can verify if the existing
+ * contents will point to a valid island.
+ */
+ return nfp_encode_basic_qdr(*addr, cpp_tgt, dest_island,
+ mode, addr40, isld1, isld0);
+
+ iid_lsb = addr40 ? 34 : 26;
+ /* <39:34> or <31:26> */
+ v64 = GENMASK_ULL(iid_lsb + 5, iid_lsb);
+ *addr &= ~v64;
+ *addr |= ((u64)dest_island << iid_lsb) & v64;
+ return 0;
+ case 1:
+ if (cpp_tgt == NFP_CPP_TARGET_QDR && !addr40)
+ return nfp_encode_basic_qdr(*addr, cpp_tgt, dest_island,
+ mode, addr40, isld1, isld0);
+
+ idx_lsb = addr40 ? 39 : 31;
+ if (dest_island == isld0) {
+ /* Only need to clear the Index bit */
+ *addr &= ~BIT_ULL(idx_lsb);
+ return 0;
+ }
+
+ if (dest_island == isld1) {
+ /* Only need to set the Index bit */
+ *addr |= BIT_ULL(idx_lsb);
+ return 0;
+ }
+
+ return -ENODEV;
+ case 2:
+ /* iid<0> = addr<30> = channel<0>
+ * channel<1> = addr<31> = Index
+ */
+ if (cpp_tgt == NFP_CPP_TARGET_QDR && !addr40)
+ /* Special case where we allow channel bits to
+ * be set before hand and with them select an island.
+ * So we need to confirm that it's at least plausible.
+ */
+ return nfp_encode_basic_qdr(*addr, cpp_tgt, dest_island,
+ mode, addr40, isld1, isld0);
+
+ /* Make sure we compare against isldN values
+ * by clearing the LSB.
+ * This is what the silicon does.
+ */
+ isld[0] &= ~1;
+ isld[1] &= ~1;
+
+ idx_lsb = addr40 ? 39 : 31;
+ iid_lsb = idx_lsb - 1;
+
+ return nfp_encode_basic_search(addr, dest_island, isld,
+ iid_lsb, idx_lsb, 2);
+ case 3:
+ if (cpp_tgt == NFP_CPP_TARGET_QDR && !addr40)
+ /* iid<0> = addr<29> = data
+ * iid<1> = addr<30> = channel<0>
+ * channel<1> = addr<31> = Index
+ */
+ return nfp_encode_basic_qdr(*addr, cpp_tgt, dest_island,
+ mode, addr40, isld1, isld0);
+
+ isld[0] &= ~3;
+ isld[1] &= ~3;
+
+ idx_lsb = addr40 ? 39 : 31;
+ iid_lsb = idx_lsb - 2;
+
+ return nfp_encode_basic_search(addr, dest_island, isld,
+ iid_lsb, idx_lsb, 4);
+ default:
+ return -EINVAL;
+ }
+}
+
+static int nfp_encode_mu(u64 *addr, int dest_island, int mode,
+ bool addr40, int isld1, int isld0)
+{
+ int iid_lsb, idx_lsb, locality_lsb;
+ int isld[2];
+ u64 v64;
+ int da;
+
+ isld[0] = isld0;
+ isld[1] = isld1;
+ locality_lsb = nfp_cppat_mu_locality_lsb(mode, addr40);
+
+ if (((*addr >> locality_lsb) & 3) == _NIC_NFP6000_MU_LOCALITY_DIRECT)
+ da = 1;
+ else
+ da = 0;
+
+ switch (mode) {
+ case 0:
+ iid_lsb = addr40 ? 32 : 24;
+ v64 = GENMASK_ULL(iid_lsb + 5, iid_lsb);
+ *addr &= ~v64;
+ *addr |= (((u64)dest_island) << iid_lsb) & v64;
+ return 0;
+ case 1:
+ if (da) {
+ iid_lsb = addr40 ? 32 : 24;
+ v64 = GENMASK_ULL(iid_lsb + 5, iid_lsb);
+ *addr &= ~v64;
+ *addr |= (((u64)dest_island) << iid_lsb) & v64;
+ return 0;
+ }
+
+ idx_lsb = addr40 ? 37 : 29;
+ if (dest_island == isld0) {
+ *addr &= ~BIT_ULL(idx_lsb);
+ return 0;
+ }
+
+ if (dest_island == isld1) {
+ *addr |= BIT_ULL(idx_lsb);
+ return 0;
+ }
+
+ return -ENODEV;
+ case 2:
+ if (da) {
+ iid_lsb = addr40 ? 32 : 24;
+ v64 = GENMASK_ULL(iid_lsb + 5, iid_lsb);
+ *addr &= ~v64;
+ *addr |= (((u64)dest_island) << iid_lsb) & v64;
+ return 0;
+ }
+
+ /* Make sure we compare against isldN values
+ * by clearing the LSB.
+ * This is what the silicon does.
+ */
+ isld[0] &= ~1;
+ isld[1] &= ~1;
+
+ idx_lsb = addr40 ? 37 : 29;
+ iid_lsb = idx_lsb - 1;
+
+ return nfp_encode_basic_search(addr, dest_island, isld,
+ iid_lsb, idx_lsb, 2);
+ case 3:
+ /* Only the EMU will use 40 bit addressing. Silently
+ * set the direct locality bit for everyone else.
+ * The SDK toolchain uses dest_island <= 0 to test
+ * for atypical address encodings to support access
+ * to local-island CTM with a 32-but address (high-locality
+ * is effewctively ignored and just used for
+ * routing to island #0).
+ */
+ if (dest_island > 0 && (dest_island < 24 || dest_island > 26)) {
+ *addr |= ((u64)_NIC_NFP6000_MU_LOCALITY_DIRECT)
+ << locality_lsb;
+ da = 1;
+ }
+
+ if (da) {
+ iid_lsb = addr40 ? 32 : 24;
+ v64 = GENMASK_ULL(iid_lsb + 5, iid_lsb);
+ *addr &= ~v64;
+ *addr |= (((u64)dest_island) << iid_lsb) & v64;
+ return 0;
+ }
+
+ isld[0] &= ~3;
+ isld[1] &= ~3;
+
+ idx_lsb = addr40 ? 37 : 29;
+ iid_lsb = idx_lsb - 2;
+
+ return nfp_encode_basic_search(addr, dest_island, isld,
+ iid_lsb, idx_lsb, 4);
+ default:
+ return -EINVAL;
+ }
+}
+
+static int nfp_cppat_addr_encode(u64 *addr, int dest_island, int cpp_tgt,
+ int mode, bool addr40, int isld1, int isld0)
+{
+ switch (cpp_tgt) {
+ case NFP_CPP_TARGET_NBI:
+ case NFP_CPP_TARGET_QDR:
+ case NFP_CPP_TARGET_ILA:
+ case NFP_CPP_TARGET_PCIE:
+ case NFP_CPP_TARGET_ARM:
+ case NFP_CPP_TARGET_CRYPTO:
+ case NFP_CPP_TARGET_CLS:
+ return nfp_encode_basic(addr, dest_island, cpp_tgt, mode,
+ addr40, isld1, isld0);
+
+ case NFP_CPP_TARGET_MU:
+ return nfp_encode_mu(addr, dest_island, mode,
+ addr40, isld1, isld0);
+
+ case NFP_CPP_TARGET_CT_XPB:
+ if (mode != 1 || addr40)
+ return -EINVAL;
+ *addr &= ~GENMASK_ULL(29, 24);
+ *addr |= ((u64)dest_island << 24) & GENMASK_ULL(29, 24);
+ return 0;
+ default:
+ return -EINVAL;
+ }
+}
+
+int nfp_target_cpp(u32 cpp_island_id, u64 cpp_island_address,
+ u32 *cpp_target_id, u64 *cpp_target_address,
+ const u32 *imb_table)
+{
+ const int island = NFP_CPP_ID_ISLAND_of(cpp_island_id);
+ const int target = NFP_CPP_ID_TARGET_of(cpp_island_id);
+ u32 imb;
+ int err;
+
+ if (target < 0 || target >= 16)
+ return -EINVAL;
+
+ if (island == 0) {
+ /* Already translated */
+ *cpp_target_id = cpp_island_id;
+ *cpp_target_address = cpp_island_address;
+ return 0;
+ }
+
+ /* CPP + Island only allowed on systems with IMB tables */
+ if (!imb_table)
+ return -EINVAL;
+
+ imb = imb_table[target];
+
+ *cpp_target_address = cpp_island_address;
+ err = nfp_cppat_addr_encode(cpp_target_address, island, target,
+ ((imb >> 13) & 7), ((imb >> 12) & 1),
+ ((imb >> 6) & 0x3f), ((imb >> 0) & 0x3f));
+ if (err)
+ return err;
+
+ *cpp_target_id = NFP_CPP_ID(target,
+ NFP_CPP_ID_ACTION_of(cpp_island_id),
+ NFP_CPP_ID_TOKEN_of(cpp_island_id));
+
+ return 0;
+}
projects / GitHub / LineageOS / android_kernel_motorola_exynos9610.git / commitdiff