// SPDX-License-Identifier: GPL-2.0+

#include <avm/pa/avm_pa.h>
#include <avm/pa/avm_pa_hw.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/printk.h>
#include <linux/in.h> /* IPPROTOs */
#include <linux/debugfs.h>
#include <linux/err.h>
#include <linux/async.h>
#include <linux/netdevice.h>
#include <linux/notifier.h>
#include <linux/atomic.h>

#include <offload/ktestdrive.h>
#include <offload/errstat.h>

#include "hwpa.h"
#include "hwpa_backend.h"

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Paul Hüber");
MODULE_DESCRIPTION("AVM compat layer for network offloading");

ASYNC_DOMAIN(hwpa_add_domain);
ASYNC_DOMAIN(hwpa_rem_domain);

/* Workaround for JZ-71010:
 * This lock makes sure that no asynchronous removal can be scheduled while
 * hwpa_session_stats() is running.
 */
DEFINE_SPINLOCK(async_rem_lock);

static struct ktd_suite *test_suite;

struct kobject *hwpa_kobj;

static struct errstat_domain_storage edom_storage;
static struct errstat_domain *edom;
static const char *hwpa_backend_rv_descriptions[] =
        HWPA_BACKEND_RV_DESC_INITIALIZER;
static atomic_t
        hwpa_backend_rv_counters[ARRAY_SIZE(hwpa_backend_rv_descriptions)];

/* avm_pa match whitelists
 *
 * valid combinations:
 * l2+l3
 * l2+l3_l3encap+l3
 * l2+l3_l2encap+l2+l3
 */
static const struct avm_pa_match_info valid_l2_matches[][AVM_PA_MAX_MATCH] = {
	HWPA_VALID_L2
};

static const struct avm_pa_match_info
	valid_l3_l2encap_matches[][AVM_PA_MAX_MATCH] = { HWPA_VALID_L3_L2ENCAP };

static const struct avm_pa_match_info
	valid_l3_l3encap_matches[][AVM_PA_MAX_MATCH] = { HWPA_VALID_L3_L3ENCAP };

static const struct avm_pa_match_info valid_l3_matches[][AVM_PA_MAX_MATCH] = {
	HWPA_VALID_L3
};

static int match_longest_seq(const struct avm_pa_match_info *needle,
			     int needle_len,
			     const struct avm_pa_match_info *haystack,
			     int haystack_size)
{
	int i;
	const struct avm_pa_match_info *haystack_match;
	int longest_match_len;

	longest_match_len = -1;

	for (i = 0; i < haystack_size; i++) {
		int j;
		int match_count;

		match_count = 0;
		haystack_match = &haystack[i * AVM_PA_MAX_MATCH];
		for (j = 0; j < needle_len &&
			    haystack_match[j].type != AVM_PA_NUM_MATCH_TYPES;
		     j++) {
			if (needle[j].type == haystack_match[j].type) {
				match_count++;
			} else {
				match_count = -1;
				break;
			}
		}
		if (haystack_match[j].type == AVM_PA_NUM_MATCH_TYPES)
			/* Haystack entry fully compared, use result. */
			longest_match_len = max(longest_match_len, match_count);
	}

	return longest_match_len;
}

static int match_traverse(const struct avm_pa_match_info *valid_matches,
			  int valid_matches_size,
			  const struct avm_pa_match_info **info,
			  int *info_len_inout)
{
	int i;

	i = match_longest_seq(*info, *info_len_inout, valid_matches,
			      valid_matches_size);
	if (i >= 0) {
		*info += i;
		*info_len_inout -= i;

		return 1;
	} else {
		return 0;
	}
}

static bool is_invalid_session_match(const struct avm_pa_pkt_match *match)
{
	const struct avm_pa_match_info *info;
	int info_len;
	bool is_matching;

	info = &match->match[0];
	info_len = match->nmatch;

#define TRAVERSE(v) match_traverse(&v[0][0], ARRAY_SIZE(v), &info, &info_len)

	/* l2 ... */
	is_matching = TRAVERSE(valid_l2_matches) &&
		      (
			      /* ... + l3 */
			      (TRAVERSE(valid_l3_matches)) ||
			      /* ... + l2encap + l2 + l3 */
			      (TRAVERSE(valid_l3_l2encap_matches) &&
			       TRAVERSE(valid_l2_matches) &&
			       TRAVERSE(valid_l3_matches)) ||
			      /* ... + l3_l3encap */
			      (TRAVERSE(valid_l3_l3encap_matches)));
#undef TRAVERSE

	return (!is_matching || info != &match->match[match->nmatch]);
}

static bool is_invalid_session_matches(const struct avm_pa_session *s)
{
	return is_invalid_session_match(&s->ingress) ||
	       is_invalid_session_match(&avm_pa_first_egress(s)->match);
}

static u16 pkttype_encap_added(u16 ingress_pkttype, u16 egress_pkttype)
{
	u16 ig_encap = ingress_pkttype & AVM_PA_PKTTYPE_IPENCAP_MASK;
	u16 eg_encap = egress_pkttype & AVM_PA_PKTTYPE_IPENCAP_MASK;

	if (ig_encap == eg_encap)
		return 0;
	else
		return eg_encap;
}

static u16 pkttype_encap_removed(u16 ingress_pkttype, u16 egress_pkttype)
{
	/* swap ingress and egress */
	return pkttype_encap_added(egress_pkttype, ingress_pkttype);
}

static bool is_invalid_session_pkttype(const struct avm_pa_session *s)
{
	u16 egress_pkttype;

	egress_pkttype = avm_pa_first_egress(s)->match.pkttype;

	if (!s->ingress.pkttype || !egress_pkttype) {
		pr_debug("pkttype not set\n");
		return true;
	}

	if ((s->ingress.pkttype | egress_pkttype) &
	    (AVM_PA_PKTTYPE_LISP | AVM_PA_PKTTYPE_GRE)) {
		pr_debug("lisp or gre\n");
		return true;
	}

	if (AVM_PA_PKTTYPE_IP_VERSION(egress_pkttype) !=
	    AVM_PA_PKTTYPE_IP_VERSION(s->ingress.pkttype)) {
		pr_debug("innermost IP version changed\n");
		return true;
	}

	if (AVM_PA_PKTTYPE_IPPROTO(egress_pkttype) !=
	    AVM_PA_PKTTYPE_IPPROTO(s->ingress.pkttype)) {
		pr_debug("innermost transport protocol changed\n");
		return true;
	}

	switch (AVM_PA_PKTTYPE_IPPROTO(egress_pkttype)) {
	case IPPROTO_TCP:
	case IPPROTO_UDP:
		break;
	default:
		pr_debug("innermost transport is neither udp nor tcp\n");
		return true;
	}

	if (pkttype_encap_added(s->ingress.pkttype, egress_pkttype) &&
	    pkttype_encap_removed(s->ingress.pkttype, egress_pkttype)) {
		pr_debug("diffenent encapsulations terminated at once\n");
		return true;
	}

	return false;
}

static bool is_invalid_session(const struct avm_pa_session *s)
{
	if (is_invalid_session_pkttype(s)) {
		pr_debug("invalid pkttypes\n");
		return true;
	} else if (is_invalid_session_matches(s)) {
		pr_debug("invalid matches\n");
		return true;
	} else
		return false;
}

static inline bool is_invalid_hw_handle(unsigned long hw_handle)
{
	return (hw_handle == hw_handle_invalid ||
		hw_handle == hw_handle_zero) ? true : false;
}

static void release_session(struct avm_pa_session *s)
{
	avm_pa_set_hw_session(s, (void *) hw_handle_zero);
}

#ifdef AVM_PA_HARDWARE_PA_HAS_PROBE_SESSION
static int hwpa_probe_session(struct avm_pa_session *avm_session)
{
	enum hwpa_backend_rv backend_rv;
	unsigned long hw_handle;

	if (is_invalid_session(avm_session))
		return AVM_PA_TX_ERROR_SESSION;

	/* Pass hw_handle down to backend -- but do not use it yet to
	 * allow an easier integration of the new avm_pa probe functionality.
	 * In future we can use it to create some preliminary session handle
	 */
	backend_rv = hwpa_backend_probe_session(avm_session, &hw_handle);
	if (backend_rv != HWPA_BACKEND_SUCCESS)
		return AVM_PA_TX_ERROR_SESSION;

	return AVM_PA_TX_SESSION_ADDED;
}
#endif

static void add_session_async(void *session_ptr, async_cookie_t cookie)
{
	enum hwpa_backend_rv backend_rv;
	unsigned long hw_handle;
	struct avm_pa_session *s = session_ptr;


	/* Wait for pending removals to finish. Avoid a situation where a
	 * newly added session is hit by an older removal request because its
	 * contents match.
	 */
	async_synchronize_cookie_domain(cookie, &hwpa_rem_domain);

	backend_rv = hwpa_backend_add_session(s, &hw_handle);
	if (errstat_track(edom, backend_rv) != HWPA_BACKEND_SUCCESS)
		return;

	BUILD_BUG_ON(sizeof(void *) < sizeof(unsigned long));
	avm_pa_set_hw_session(s, (void *)hw_handle);
}

static int hwpa_add_session(struct avm_pa_session *avm_session)
{
#ifndef AVM_PA_HARDWARE_PA_HAS_PROBE_SESSION
	if (is_invalid_session(avm_session))
		return AVM_PA_TX_ERROR_SESSION;
#endif

	avm_pa_set_hw_session(avm_session, (void *)hw_handle_invalid);

	async_schedule_domain(add_session_async, avm_session, &hwpa_add_domain);

	return AVM_PA_TX_SESSION_ADDED;
}

static void remove_session_async(void *session_ptr, async_cookie_t cookie)
{
	unsigned long hw_handle;
	struct avm_pa_session *avm_session = session_ptr;

	/* Make sure the session was added. */
	async_synchronize_cookie_domain(cookie, &hwpa_add_domain);

	hw_handle = (unsigned long)avm_pa_get_hw_session(avm_session);

	if (hw_handle != hw_handle_invalid)
		errstat_track(edom, hwpa_backend_rem_session(hw_handle));

	release_session(avm_session);
}

static int hwpa_remove_session(struct avm_pa_session *avm_session)
{
	/* Workaround for JZ-71010:
	 * Protect hwpa_session_stats().
	 */
	spin_lock_bh(&async_rem_lock);
	async_schedule_domain(remove_session_async, avm_session,
	                      &hwpa_rem_domain);
	spin_unlock_bh(&async_rem_lock);

	return AVM_PA_TX_SESSION_ADDED;
}

static int hwpa_session_stats(struct avm_pa_session *avm_session,
			      struct avm_pa_session_stats *stats)
{
	unsigned long hw_handle;
	int rv = -1;

	/* Workaround for JZ-71010:
	 * Do not race with remove_session_async(). Make sure it's neither
	 * scheduled nor will be scheduled as long as hw_handle is being used
	 * here.
	 * TODO: bundle/replace hw_handle with a kobj to use per-session
	 * reference counting. This will likely involve implicit removal of a
	 * backend session and therefore a change of the backend interface.
	 */
	spin_lock_bh(&async_rem_lock);
	if (!list_empty(&hwpa_rem_domain.pending)) {
		memset(stats, 0, sizeof(*stats));
		goto err_unlock;
	}

	hw_handle = (unsigned long)avm_pa_get_hw_session(avm_session);

	/* Ignore invalid handles for pending add
	 * Also need to check for NULL here to catch following caveat:
	 * 1. CPU0: hwpa_session_stats just enters or waits for lock
	 * 2. CPU1: hwpa_remove_session_async finishes for this very session
	 * 3. CPU0: hwpa_session_stats gets the lock; no pending removes;
	 *          finds hw_handle NULL
	 */
	if (is_invalid_hw_handle(hw_handle))
		goto err_unlock;

	if (errstat_track(edom, hwpa_backend_stats(hw_handle, stats)) ==
	    HWPA_BACKEND_SUCCESS)
		rv = 0;

err_unlock:
	spin_unlock_bh(&async_rem_lock);
	return rv;
}

__attribute__((weak)) enum hwpa_backend_rv backend_activate_hw(avm_pid_handle pid_handle)
{
	return HWPA_BACKEND_SUCCESS;
}

static int notifier_fn(struct notifier_block *nb, unsigned long action, void *data) {
	struct netdev_notifier_info *info = data;
	avm_pid_handle pid;

	switch (action) {
	case NETDEV_UP:
		pid = AVM_PA_DEVINFO(info->dev)->pid_handle;
		if (pid && backend_activate_hw(pid) == HWPA_BACKEND_SUCCESS)
			avm_pa_pid_activate_hw_accelaration(pid);
		return NOTIFY_OK;
	default:
		return NOTIFY_DONE;
	}
}

struct notifier_block notifier = { .notifier_call = notifier_fn };

__attribute__((weak)) int try_to_accelerate(avm_pid_handle pid_handle,
                                            struct sk_buff *skb)
{
	return AVM_PA_RX_OK;
}

static struct avm_hardware_pa hw_pa = {
#ifdef AVM_PA_HARDWARE_PA_HAS_PROBE_SESSION
	.probe_session = hwpa_probe_session,
#endif
	.add_session = hwpa_add_session,
	.remove_session = hwpa_remove_session,
	.session_stats = hwpa_session_stats,

	.try_to_accelerate = try_to_accelerate,
};

/* Tests */

static ktd_ret_t is_invalid_session_pkttype_test(void *arg)
{
	int i;
	static const struct avm_pa_session valid_sessions[] = {
		/* basic tcp */
		{ .static_egress = { .match = { .pkttype = AVM_PA_PKTTYPE_IPV4 |
							   IPPROTO_TCP } },
		  .ingress = { .pkttype = AVM_PA_PKTTYPE_IPV4 | IPPROTO_TCP } },
	};
	static const struct avm_pa_session invalid_sessions[] = {
		/* empty */
		{ .negress = 0 },
		/* changed ip version */
		{ .static_egress = { .match = { .pkttype = AVM_PA_PKTTYPE_IPV4 |
							   IPPROTO_TCP } },
		  .ingress = { .pkttype = AVM_PA_PKTTYPE_IPV6 | IPPROTO_TCP } },
		/* changed proto */
		{ .static_egress = { .match = { .pkttype = AVM_PA_PKTTYPE_IPV4 |
							   IPPROTO_UDP } },
		  .ingress = { .pkttype = AVM_PA_PKTTYPE_IPV4 | IPPROTO_TCP } },
		/* gre */
		{ .static_egress = { .match = { .pkttype = AVM_PA_PKTTYPE_IPV4 |
							   IPPROTO_TCP } },
		  .ingress = { .pkttype = AVM_PA_PKTTYPE_IPV4 | IPPROTO_TCP |
					  AVM_PA_PKTTYPE_GRE } },
		/* changed ip encap version */
		{ .static_egress = { .match = { .pkttype =
							AVM_PA_PKTTYPE_IPV4 |
							IPPROTO_TCP |
							AVM_PA_PKTTYPE_IPV6ENCAP } },
		  .ingress = { .pkttype = AVM_PA_PKTTYPE_IPV4 | IPPROTO_TCP |
					  AVM_PA_PKTTYPE_IPV4ENCAP } },
		/* unsupported ip proto */
		{ .static_egress = { .match = { .pkttype = AVM_PA_PKTTYPE_IPV4 |
							   IPPROTO_PUP } },
		  .ingress = { .pkttype = AVM_PA_PKTTYPE_IPV4 | IPPROTO_PUP } },
	};

	for (i = 0; i < ARRAY_SIZE(valid_sessions); i++) {
		KTD_EXPECT(!is_invalid_session_pkttype(&valid_sessions[i]));
	}

	for (i = 0; i < ARRAY_SIZE(invalid_sessions); i++) {
		KTD_EXPECT(is_invalid_session_pkttype(&invalid_sessions[i]));
	}

	return KTD_PASSED;
}

static unsigned char _copy_valid_minfos(struct avm_pa_match_info *dst,
					const struct avm_pa_match_info *src)
{
	unsigned char n;

	for (n = 0; src[n].type != AVM_PA_NUM_MATCH_TYPES; n++) {
		dst[n] = src[n];
	}
	return n;
}

static ktd_ret_t is_invalid_session_match_test(void *arg)
{
	int i;
	unsigned int total_tested;
	unsigned char *nmatch;
	struct avm_pa_match_info *minfo;
	struct avm_pa_pkt_match valid_match;
	static const struct avm_pa_pkt_match invalid_matches[] = {
		/* empty */
		{ .match = { {} }, .nmatch = 0 },
		/* l2 only */
		{ .match = { { .type = AVM_PA_ETH } }, .nmatch = 1 },
		/* repeated l2 */
		{ .match = { { .type = AVM_PA_ETH }, { .type = AVM_PA_ETH } },
		  .nmatch = 2 },
		/* l3encap with inner l2 */
		{ .match = { { .type = AVM_PA_ETH },
			     { .type = AVM_PA_IPV6 },
			     { .type = AVM_PA_ETH },
			     { .type = AVM_PA_IPV4 },
			     { .type = AVM_PA_PORTS } },
		  .nmatch = 5 },
		/* l2encap with inner l3 */
		{ .match = { { .type = AVM_PA_ETH },
			     { .type = AVM_PA_IPV4 },
			     { .type = AVM_PA_L2TP },
			     { .type = AVM_PA_IPV4 },
			     { .type = AVM_PA_PORTS } },
		  .nmatch = 5 },
		/* nested encap */
		{ .match = { { .type = AVM_PA_ETH },
			     { .type = AVM_PA_IPV4 },
			     { .type = AVM_PA_L2TP },
			     { .type = AVM_PA_ETH },
			     { .type = AVM_PA_IPV4 },
			     { .type = AVM_PA_L2TP },
			     { .type = AVM_PA_ETH },
			     { .type = AVM_PA_IPV4 },
			     { .type = AVM_PA_PORTS } },
		  .nmatch = 9 },
	};

	/* exhaustive check of valid matches */
	total_tested = 0;
	minfo = &valid_match.match[0];
	nmatch = &valid_match.nmatch;

	/* only expect nmatch and match[] to be consistent */
	memset(&valid_match, 0xff, sizeof(valid_match));
	*nmatch = 0;

	for (i = 0; i < ARRAY_SIZE(valid_l2_matches); i++) {
		int j;
		unsigned char local_n;

		local_n = _copy_valid_minfos(&minfo[*nmatch],
					     &valid_l2_matches[i][0]);
		*nmatch += local_n;

		for (j = 0; j < ARRAY_SIZE(valid_l3_l2encap_matches); j++) {
			int k;
			unsigned char local_n;

			local_n = _copy_valid_minfos(
				&minfo[*nmatch],
				&valid_l3_l2encap_matches[j][0]);
			*nmatch += local_n;

			for (k = 0; k < ARRAY_SIZE(valid_l2_matches); k++) {
				int l;
				unsigned char local_n;

				local_n = _copy_valid_minfos(
					&minfo[*nmatch],
					&valid_l2_matches[k][0]);
				*nmatch += local_n;

				for (l = 0; l < ARRAY_SIZE(valid_l3_matches);
				     l++) {
					unsigned char local_n;

					local_n = _copy_valid_minfos(
						&minfo[*nmatch],
						&valid_l3_matches[l][0]);

					*nmatch += local_n;

					KTD_EXPECT(*nmatch < AVM_PA_MAX_MATCH);
					KTD_EXPECT(!is_invalid_session_match(
						&valid_match));
					total_tested++;

					*nmatch -= local_n;
				}

				*nmatch -= local_n;
			}

			*nmatch -= local_n;
		}

		for (j = 0; j < ARRAY_SIZE(valid_l3_l3encap_matches); j++) {
			int k;
			unsigned char local_n;

			local_n = _copy_valid_minfos(
				&minfo[*nmatch],
				&valid_l3_l3encap_matches[j][0]);
			*nmatch += local_n;

			for (k = 0; k < ARRAY_SIZE(valid_l3_matches); k++) {
				unsigned char local_n;

				local_n = _copy_valid_minfos(
					&minfo[*nmatch],
					&valid_l3_matches[k][0]);

				*nmatch += local_n;

				KTD_EXPECT(*nmatch < AVM_PA_MAX_MATCH);
				KTD_EXPECT(!is_invalid_session_match(
					&valid_match));
				total_tested++;

				*nmatch -= local_n;
			}

			*nmatch -= local_n;
		}

		for (j = 0; j < ARRAY_SIZE(valid_l3_matches); j++) {
			unsigned char local_n;

			local_n = _copy_valid_minfos(&minfo[*nmatch],
						     &valid_l3_matches[j][0]);

			*nmatch += local_n;

			KTD_EXPECT(*nmatch < AVM_PA_MAX_MATCH);
			KTD_EXPECT(!is_invalid_session_match(&valid_match));
			total_tested++;

			*nmatch -= local_n;
		}

		*nmatch -= local_n;
	}

	/* make sure enough cases were hit */
	KTD_EXPECT(total_tested != 0);
	total_tested -= ARRAY_SIZE(valid_l2_matches) *
			ARRAY_SIZE(valid_l3_l2encap_matches) *
			ARRAY_SIZE(valid_l2_matches) *
			ARRAY_SIZE(valid_l3_matches);

	total_tested -= ARRAY_SIZE(valid_l2_matches) *
			ARRAY_SIZE(valid_l3_l3encap_matches) *
			ARRAY_SIZE(valid_l3_matches);

	total_tested -=
		ARRAY_SIZE(valid_l2_matches) * ARRAY_SIZE(valid_l3_matches);
	KTD_EXPECT(total_tested == 0);

	/* test some constructed pathological cases */
	for (i = 0; i < ARRAY_SIZE(invalid_matches); i++) {
		KTD_EXPECT(is_invalid_session_match(&invalid_matches[i]));
	}

	return KTD_PASSED;
}

int __init hwpa_init(void)
{
	struct hwpa_backend_config hw_pa_config = {0};

	edom = errstat_domain_init(&edom_storage, hwpa_backend_rv_descriptions,
	                           hwpa_backend_rv_counters,
	                           ARRAY_SIZE(hwpa_backend_rv_counters));

	hwpa_kobj = kobject_create_and_add(THIS_MODULE->name, kernel_kobj);
	if (!hwpa_kobj)
		return -1;

	errstat_sysfs_attach(edom, hwpa_kobj, "backend_errors");

	test_suite = ktd_suite_create(THIS_MODULE->name);

	hwpa_backend_init(&hw_pa_config);

	register_netdevice_notifier(&notifier);

	ktd_register(test_suite, "is_invalid_session_match",
		     is_invalid_session_match_test, NULL);
	ktd_register(test_suite, "is_invalid_session_pkttype",
		     is_invalid_session_pkttype_test, NULL);

	hw_pa.alloc_rx_channel = hw_pa_config.alloc_rx_channel;
	hw_pa.alloc_tx_channel = hw_pa_config.alloc_tx_channel;
	hw_pa.free_rx_channel = hw_pa_config.free_rx_channel;
	hw_pa.free_tx_channel = hw_pa_config.free_tx_channel;
	hw_pa.flags |= hw_pa_config.flags & HWPA_PLATFORM_SPECIFIC_FLAGS_MASK;

	avm_pa_register_hardware_pa(&hw_pa);

	return 0;
}

void __exit hwpa_exit(void)
{
	ktd_suite_destroy(test_suite);

	unregister_netdevice_notifier(&notifier);

#ifdef AVM_PA_UNREGISTER_HARDWARE_PA_SYNC
	avm_pa_unregister_hardware_pa_sync(&hw_pa);
#else
	avm_pa_register_hardware_pa(NULL);
#endif

	async_synchronize_full_domain(&hwpa_add_domain);
	async_synchronize_full_domain(&hwpa_rem_domain);
	async_unregister_domain(&hwpa_add_domain);
	async_unregister_domain(&hwpa_rem_domain);

	hwpa_backend_exit();

	errstat_sysfs_detach(edom);

	kobject_put(hwpa_kobj);
}

module_init(hwpa_init);
module_exit(hwpa_exit);