1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
2 /* Copyright (c) 2018 Mellanox Technologies. All rights reserved */
4 #include <linux/bitmap.h>
5 #include <linux/errno.h>
6 #include <linux/genalloc.h>
8 #include <linux/kernel.h>
9 #include <linux/list.h>
10 #include <linux/rhashtable.h>
11 #include <linux/rtnetlink.h>
12 #include <linux/slab.h>
17 #include "spectrum_acl_tcam.h"
19 /* gen_pool_alloc() returns 0 when allocation fails, so use an offset */
20 #define MLXSW_SP_ACL_ERP_GENALLOC_OFFSET 0x100
21 #define MLXSW_SP_ACL_ERP_MAX_PER_REGION 16
23 struct mlxsw_sp_acl_erp_core {
24 unsigned int erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_MAX + 1];
25 struct gen_pool *erp_tables;
26 struct mlxsw_sp *mlxsw_sp;
27 unsigned int num_erp_banks;
30 struct mlxsw_sp_acl_erp_key {
31 char mask[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN];
35 struct mlxsw_sp_acl_erp {
36 struct mlxsw_sp_acl_erp_key key;
40 DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN);
41 struct list_head list;
42 struct rhash_head ht_node;
43 struct mlxsw_sp_acl_erp_table *erp_table;
46 struct mlxsw_sp_acl_erp_master_mask {
47 DECLARE_BITMAP(bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN);
48 unsigned int count[MLXSW_SP_ACL_TCAM_MASK_LEN];
51 struct mlxsw_sp_acl_erp_table {
52 struct mlxsw_sp_acl_erp_master_mask master_mask;
53 DECLARE_BITMAP(erp_id_bitmap, MLXSW_SP_ACL_ERP_MAX_PER_REGION);
54 DECLARE_BITMAP(erp_index_bitmap, MLXSW_SP_ACL_ERP_MAX_PER_REGION);
55 struct list_head atcam_erps_list;
56 struct rhashtable erp_ht;
57 struct mlxsw_sp_acl_erp_core *erp_core;
58 struct mlxsw_sp_acl_atcam_region *aregion;
59 const struct mlxsw_sp_acl_erp_table_ops *ops;
60 unsigned long base_index;
61 unsigned int num_atcam_erps;
62 unsigned int num_max_atcam_erps;
63 unsigned int num_ctcam_erps;
66 static const struct rhashtable_params mlxsw_sp_acl_erp_ht_params = {
67 .key_len = sizeof(struct mlxsw_sp_acl_erp_key),
68 .key_offset = offsetof(struct mlxsw_sp_acl_erp, key),
69 .head_offset = offsetof(struct mlxsw_sp_acl_erp, ht_node),
72 struct mlxsw_sp_acl_erp_table_ops {
73 struct mlxsw_sp_acl_erp *
74 (*erp_create)(struct mlxsw_sp_acl_erp_table *erp_table,
75 struct mlxsw_sp_acl_erp_key *key);
76 void (*erp_destroy)(struct mlxsw_sp_acl_erp_table *erp_table,
77 struct mlxsw_sp_acl_erp *erp);
80 static struct mlxsw_sp_acl_erp *
81 mlxsw_sp_acl_erp_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
82 struct mlxsw_sp_acl_erp_key *key);
84 mlxsw_sp_acl_erp_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
85 struct mlxsw_sp_acl_erp *erp);
86 static struct mlxsw_sp_acl_erp *
87 mlxsw_sp_acl_erp_second_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
88 struct mlxsw_sp_acl_erp_key *key);
90 mlxsw_sp_acl_erp_second_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
91 struct mlxsw_sp_acl_erp *erp);
92 static struct mlxsw_sp_acl_erp *
93 mlxsw_sp_acl_erp_first_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
94 struct mlxsw_sp_acl_erp_key *key);
96 mlxsw_sp_acl_erp_first_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
97 struct mlxsw_sp_acl_erp *erp);
99 mlxsw_sp_acl_erp_no_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
100 struct mlxsw_sp_acl_erp *erp);
102 static const struct mlxsw_sp_acl_erp_table_ops erp_multiple_masks_ops = {
103 .erp_create = mlxsw_sp_acl_erp_mask_create,
104 .erp_destroy = mlxsw_sp_acl_erp_mask_destroy,
107 static const struct mlxsw_sp_acl_erp_table_ops erp_two_masks_ops = {
108 .erp_create = mlxsw_sp_acl_erp_mask_create,
109 .erp_destroy = mlxsw_sp_acl_erp_second_mask_destroy,
112 static const struct mlxsw_sp_acl_erp_table_ops erp_single_mask_ops = {
113 .erp_create = mlxsw_sp_acl_erp_second_mask_create,
114 .erp_destroy = mlxsw_sp_acl_erp_first_mask_destroy,
117 static const struct mlxsw_sp_acl_erp_table_ops erp_no_mask_ops = {
118 .erp_create = mlxsw_sp_acl_erp_first_mask_create,
119 .erp_destroy = mlxsw_sp_acl_erp_no_mask_destroy,
122 bool mlxsw_sp_acl_erp_is_ctcam_erp(const struct mlxsw_sp_acl_erp *erp)
124 return erp->key.ctcam;
127 u8 mlxsw_sp_acl_erp_id(const struct mlxsw_sp_acl_erp *erp)
133 mlxsw_sp_acl_erp_table_entry_size(const struct mlxsw_sp_acl_erp_table *erp_table)
135 struct mlxsw_sp_acl_atcam_region *aregion = erp_table->aregion;
136 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core;
138 return erp_core->erpt_entries_size[aregion->type];
141 static int mlxsw_sp_acl_erp_id_get(struct mlxsw_sp_acl_erp_table *erp_table,
146 id = find_first_zero_bit(erp_table->erp_id_bitmap,
147 MLXSW_SP_ACL_ERP_MAX_PER_REGION);
148 if (id < MLXSW_SP_ACL_ERP_MAX_PER_REGION) {
149 __set_bit(id, erp_table->erp_id_bitmap);
157 static void mlxsw_sp_acl_erp_id_put(struct mlxsw_sp_acl_erp_table *erp_table,
160 __clear_bit(id, erp_table->erp_id_bitmap);
164 mlxsw_sp_acl_erp_master_mask_bit_set(unsigned long bit,
165 struct mlxsw_sp_acl_erp_master_mask *mask)
167 if (mask->count[bit]++ == 0)
168 __set_bit(bit, mask->bitmap);
172 mlxsw_sp_acl_erp_master_mask_bit_clear(unsigned long bit,
173 struct mlxsw_sp_acl_erp_master_mask *mask)
175 if (--mask->count[bit] == 0)
176 __clear_bit(bit, mask->bitmap);
180 mlxsw_sp_acl_erp_master_mask_update(struct mlxsw_sp_acl_erp_table *erp_table)
182 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region;
183 struct mlxsw_sp *mlxsw_sp = region->mlxsw_sp;
184 char percr_pl[MLXSW_REG_PERCR_LEN];
187 mlxsw_reg_percr_pack(percr_pl, region->id);
188 master_mask = mlxsw_reg_percr_master_mask_data(percr_pl);
189 bitmap_to_arr32((u32 *) master_mask, erp_table->master_mask.bitmap,
190 MLXSW_SP_ACL_TCAM_MASK_LEN);
192 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(percr), percr_pl);
196 mlxsw_sp_acl_erp_master_mask_set(struct mlxsw_sp_acl_erp_table *erp_table,
197 const struct mlxsw_sp_acl_erp *erp)
202 for_each_set_bit(bit, erp->mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN)
203 mlxsw_sp_acl_erp_master_mask_bit_set(bit,
204 &erp_table->master_mask);
206 err = mlxsw_sp_acl_erp_master_mask_update(erp_table);
208 goto err_master_mask_update;
212 err_master_mask_update:
213 for_each_set_bit(bit, erp->mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN)
214 mlxsw_sp_acl_erp_master_mask_bit_clear(bit,
215 &erp_table->master_mask);
220 mlxsw_sp_acl_erp_master_mask_clear(struct mlxsw_sp_acl_erp_table *erp_table,
221 const struct mlxsw_sp_acl_erp *erp)
226 for_each_set_bit(bit, erp->mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN)
227 mlxsw_sp_acl_erp_master_mask_bit_clear(bit,
228 &erp_table->master_mask);
230 err = mlxsw_sp_acl_erp_master_mask_update(erp_table);
232 goto err_master_mask_update;
236 err_master_mask_update:
237 for_each_set_bit(bit, erp->mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN)
238 mlxsw_sp_acl_erp_master_mask_bit_set(bit,
239 &erp_table->master_mask);
243 static struct mlxsw_sp_acl_erp *
244 mlxsw_sp_acl_erp_generic_create(struct mlxsw_sp_acl_erp_table *erp_table,
245 struct mlxsw_sp_acl_erp_key *key)
247 struct mlxsw_sp_acl_erp *erp;
250 erp = kzalloc(sizeof(*erp), GFP_KERNEL);
252 return ERR_PTR(-ENOMEM);
254 err = mlxsw_sp_acl_erp_id_get(erp_table, &erp->id);
258 memcpy(&erp->key, key, sizeof(*key));
259 bitmap_from_arr32(erp->mask_bitmap, (u32 *) key->mask,
260 MLXSW_SP_ACL_TCAM_MASK_LEN);
261 list_add(&erp->list, &erp_table->atcam_erps_list);
262 refcount_set(&erp->refcnt, 1);
263 erp_table->num_atcam_erps++;
264 erp->erp_table = erp_table;
266 err = mlxsw_sp_acl_erp_master_mask_set(erp_table, erp);
268 goto err_master_mask_set;
270 err = rhashtable_insert_fast(&erp_table->erp_ht, &erp->ht_node,
271 mlxsw_sp_acl_erp_ht_params);
273 goto err_rhashtable_insert;
277 err_rhashtable_insert:
278 mlxsw_sp_acl_erp_master_mask_clear(erp_table, erp);
280 erp_table->num_atcam_erps--;
281 list_del(&erp->list);
282 mlxsw_sp_acl_erp_id_put(erp_table, erp->id);
289 mlxsw_sp_acl_erp_generic_destroy(struct mlxsw_sp_acl_erp *erp)
291 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table;
293 rhashtable_remove_fast(&erp_table->erp_ht, &erp->ht_node,
294 mlxsw_sp_acl_erp_ht_params);
295 mlxsw_sp_acl_erp_master_mask_clear(erp_table, erp);
296 erp_table->num_atcam_erps--;
297 list_del(&erp->list);
298 mlxsw_sp_acl_erp_id_put(erp_table, erp->id);
303 mlxsw_sp_acl_erp_table_alloc(struct mlxsw_sp_acl_erp_core *erp_core,
304 unsigned int num_erps,
305 enum mlxsw_sp_acl_atcam_region_type region_type,
306 unsigned long *p_index)
308 unsigned int num_rows, entry_size;
310 /* We only allow allocations of entire rows */
311 if (num_erps % erp_core->num_erp_banks != 0)
314 entry_size = erp_core->erpt_entries_size[region_type];
315 num_rows = num_erps / erp_core->num_erp_banks;
317 *p_index = gen_pool_alloc(erp_core->erp_tables, num_rows * entry_size);
320 *p_index -= MLXSW_SP_ACL_ERP_GENALLOC_OFFSET;
326 mlxsw_sp_acl_erp_table_free(struct mlxsw_sp_acl_erp_core *erp_core,
327 unsigned int num_erps,
328 enum mlxsw_sp_acl_atcam_region_type region_type,
331 unsigned long base_index;
332 unsigned int entry_size;
335 entry_size = erp_core->erpt_entries_size[region_type];
336 base_index = index + MLXSW_SP_ACL_ERP_GENALLOC_OFFSET;
337 size = num_erps / erp_core->num_erp_banks * entry_size;
338 gen_pool_free(erp_core->erp_tables, base_index, size);
341 static struct mlxsw_sp_acl_erp *
342 mlxsw_sp_acl_erp_table_master_rp(struct mlxsw_sp_acl_erp_table *erp_table)
344 if (!list_is_singular(&erp_table->atcam_erps_list))
347 return list_first_entry(&erp_table->atcam_erps_list,
348 struct mlxsw_sp_acl_erp, list);
351 static int mlxsw_sp_acl_erp_index_get(struct mlxsw_sp_acl_erp_table *erp_table,
356 index = find_first_zero_bit(erp_table->erp_index_bitmap,
357 erp_table->num_max_atcam_erps);
358 if (index < erp_table->num_max_atcam_erps) {
359 __set_bit(index, erp_table->erp_index_bitmap);
367 static void mlxsw_sp_acl_erp_index_put(struct mlxsw_sp_acl_erp_table *erp_table,
370 __clear_bit(index, erp_table->erp_index_bitmap);
374 mlxsw_sp_acl_erp_table_locate(const struct mlxsw_sp_acl_erp_table *erp_table,
375 const struct mlxsw_sp_acl_erp *erp,
376 u8 *p_erpt_bank, u8 *p_erpt_index)
378 unsigned int entry_size = mlxsw_sp_acl_erp_table_entry_size(erp_table);
379 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core;
382 *p_erpt_bank = erp->index % erp_core->num_erp_banks;
383 row = erp->index / erp_core->num_erp_banks;
384 *p_erpt_index = erp_table->base_index + row * entry_size;
388 mlxsw_sp_acl_erp_table_erp_add(struct mlxsw_sp_acl_erp_table *erp_table,
389 struct mlxsw_sp_acl_erp *erp)
391 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;
392 enum mlxsw_reg_perpt_key_size key_size;
393 char perpt_pl[MLXSW_REG_PERPT_LEN];
394 u8 erpt_bank, erpt_index;
396 mlxsw_sp_acl_erp_table_locate(erp_table, erp, &erpt_bank, &erpt_index);
397 key_size = (enum mlxsw_reg_perpt_key_size) erp_table->aregion->type;
398 mlxsw_reg_perpt_pack(perpt_pl, erpt_bank, erpt_index, key_size, erp->id,
399 0, erp_table->base_index, erp->index,
401 mlxsw_reg_perpt_erp_vector_pack(perpt_pl, erp_table->erp_index_bitmap,
402 MLXSW_SP_ACL_ERP_MAX_PER_REGION);
403 mlxsw_reg_perpt_erp_vector_set(perpt_pl, erp->index, true);
404 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(perpt), perpt_pl);
407 static void mlxsw_sp_acl_erp_table_erp_del(struct mlxsw_sp_acl_erp *erp)
409 char empty_mask[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN] = { 0 };
410 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table;
411 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;
412 enum mlxsw_reg_perpt_key_size key_size;
413 char perpt_pl[MLXSW_REG_PERPT_LEN];
414 u8 erpt_bank, erpt_index;
416 mlxsw_sp_acl_erp_table_locate(erp_table, erp, &erpt_bank, &erpt_index);
417 key_size = (enum mlxsw_reg_perpt_key_size) erp_table->aregion->type;
418 mlxsw_reg_perpt_pack(perpt_pl, erpt_bank, erpt_index, key_size, erp->id,
419 0, erp_table->base_index, erp->index, empty_mask);
420 mlxsw_reg_perpt_erp_vector_pack(perpt_pl, erp_table->erp_index_bitmap,
421 MLXSW_SP_ACL_ERP_MAX_PER_REGION);
422 mlxsw_reg_perpt_erp_vector_set(perpt_pl, erp->index, false);
423 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(perpt), perpt_pl);
427 mlxsw_sp_acl_erp_table_enable(struct mlxsw_sp_acl_erp_table *erp_table,
430 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region;
431 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;
432 char pererp_pl[MLXSW_REG_PERERP_LEN];
434 mlxsw_reg_pererp_pack(pererp_pl, region->id, ctcam_le, true, 0,
435 erp_table->base_index, 0);
436 mlxsw_reg_pererp_erp_vector_pack(pererp_pl, erp_table->erp_index_bitmap,
437 MLXSW_SP_ACL_ERP_MAX_PER_REGION);
439 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl);
443 mlxsw_sp_acl_erp_table_disable(struct mlxsw_sp_acl_erp_table *erp_table)
445 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region;
446 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;
447 char pererp_pl[MLXSW_REG_PERERP_LEN];
448 struct mlxsw_sp_acl_erp *master_rp;
450 master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table);
451 /* It is possible we do not have a master RP when we disable the
452 * table when there are no rules in the A-TCAM and the last C-TCAM
455 mlxsw_reg_pererp_pack(pererp_pl, region->id, false, false, 0, 0,
456 master_rp ? master_rp->id : 0);
457 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl);
461 mlxsw_sp_acl_erp_table_relocate(struct mlxsw_sp_acl_erp_table *erp_table)
463 struct mlxsw_sp_acl_erp *erp;
466 list_for_each_entry(erp, &erp_table->atcam_erps_list, list) {
467 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, erp);
469 goto err_table_erp_add;
475 list_for_each_entry_continue_reverse(erp, &erp_table->atcam_erps_list,
477 mlxsw_sp_acl_erp_table_erp_del(erp);
482 mlxsw_sp_acl_erp_table_expand(struct mlxsw_sp_acl_erp_table *erp_table)
484 unsigned int num_erps, old_num_erps = erp_table->num_max_atcam_erps;
485 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core;
486 unsigned long old_base_index = erp_table->base_index;
487 bool ctcam_le = erp_table->num_ctcam_erps > 0;
490 if (erp_table->num_atcam_erps < erp_table->num_max_atcam_erps)
493 if (erp_table->num_max_atcam_erps == MLXSW_SP_ACL_ERP_MAX_PER_REGION)
496 num_erps = old_num_erps + erp_core->num_erp_banks;
497 err = mlxsw_sp_acl_erp_table_alloc(erp_core, num_erps,
498 erp_table->aregion->type,
499 &erp_table->base_index);
502 erp_table->num_max_atcam_erps = num_erps;
504 err = mlxsw_sp_acl_erp_table_relocate(erp_table);
506 goto err_table_relocate;
508 err = mlxsw_sp_acl_erp_table_enable(erp_table, ctcam_le);
510 goto err_table_enable;
512 mlxsw_sp_acl_erp_table_free(erp_core, old_num_erps,
513 erp_table->aregion->type, old_base_index);
519 erp_table->num_max_atcam_erps = old_num_erps;
520 mlxsw_sp_acl_erp_table_free(erp_core, num_erps,
521 erp_table->aregion->type,
522 erp_table->base_index);
523 erp_table->base_index = old_base_index;
528 mlxsw_sp_acl_erp_region_table_trans(struct mlxsw_sp_acl_erp_table *erp_table)
530 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core;
531 struct mlxsw_sp_acl_erp *master_rp;
534 /* Initially, allocate a single eRP row. Expand later as needed */
535 err = mlxsw_sp_acl_erp_table_alloc(erp_core, erp_core->num_erp_banks,
536 erp_table->aregion->type,
537 &erp_table->base_index);
540 erp_table->num_max_atcam_erps = erp_core->num_erp_banks;
542 /* Transition the sole RP currently configured (the master RP)
545 master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table);
548 goto err_table_master_rp;
551 /* Maintain the same eRP bank for the master RP, so that we
552 * wouldn't need to update the bloom filter
554 master_rp->index = master_rp->index % erp_core->num_erp_banks;
555 __set_bit(master_rp->index, erp_table->erp_index_bitmap);
557 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, master_rp);
559 goto err_table_master_rp_add;
561 err = mlxsw_sp_acl_erp_table_enable(erp_table, false);
563 goto err_table_enable;
568 mlxsw_sp_acl_erp_table_erp_del(master_rp);
569 err_table_master_rp_add:
570 __clear_bit(master_rp->index, erp_table->erp_index_bitmap);
572 mlxsw_sp_acl_erp_table_free(erp_core, erp_table->num_max_atcam_erps,
573 erp_table->aregion->type,
574 erp_table->base_index);
579 mlxsw_sp_acl_erp_region_master_mask_trans(struct mlxsw_sp_acl_erp_table *erp_table)
581 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core;
582 struct mlxsw_sp_acl_erp *master_rp;
584 mlxsw_sp_acl_erp_table_disable(erp_table);
585 master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table);
588 mlxsw_sp_acl_erp_table_erp_del(master_rp);
589 __clear_bit(master_rp->index, erp_table->erp_index_bitmap);
590 mlxsw_sp_acl_erp_table_free(erp_core, erp_table->num_max_atcam_erps,
591 erp_table->aregion->type,
592 erp_table->base_index);
596 mlxsw_sp_acl_erp_region_erp_add(struct mlxsw_sp_acl_erp_table *erp_table,
597 struct mlxsw_sp_acl_erp *erp)
599 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region;
600 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;
601 bool ctcam_le = erp_table->num_ctcam_erps > 0;
602 char pererp_pl[MLXSW_REG_PERERP_LEN];
604 mlxsw_reg_pererp_pack(pererp_pl, region->id, ctcam_le, true, 0,
605 erp_table->base_index, 0);
606 mlxsw_reg_pererp_erp_vector_pack(pererp_pl, erp_table->erp_index_bitmap,
607 MLXSW_SP_ACL_ERP_MAX_PER_REGION);
608 mlxsw_reg_pererp_erpt_vector_set(pererp_pl, erp->index, true);
610 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl);
613 static void mlxsw_sp_acl_erp_region_erp_del(struct mlxsw_sp_acl_erp *erp)
615 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table;
616 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region;
617 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;
618 bool ctcam_le = erp_table->num_ctcam_erps > 0;
619 char pererp_pl[MLXSW_REG_PERERP_LEN];
621 mlxsw_reg_pererp_pack(pererp_pl, region->id, ctcam_le, true, 0,
622 erp_table->base_index, 0);
623 mlxsw_reg_pererp_erp_vector_pack(pererp_pl, erp_table->erp_index_bitmap,
624 MLXSW_SP_ACL_ERP_MAX_PER_REGION);
625 mlxsw_reg_pererp_erpt_vector_set(pererp_pl, erp->index, false);
627 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl);
631 mlxsw_sp_acl_erp_region_ctcam_enable(struct mlxsw_sp_acl_erp_table *erp_table)
633 /* No need to re-enable lookup in the C-TCAM */
634 if (erp_table->num_ctcam_erps > 1)
637 return mlxsw_sp_acl_erp_table_enable(erp_table, true);
641 mlxsw_sp_acl_erp_region_ctcam_disable(struct mlxsw_sp_acl_erp_table *erp_table)
643 /* Only disable C-TCAM lookup when last C-TCAM eRP is deleted */
644 if (erp_table->num_ctcam_erps > 1)
647 mlxsw_sp_acl_erp_table_enable(erp_table, false);
651 mlxsw_sp_acl_erp_ctcam_table_ops_set(struct mlxsw_sp_acl_erp_table *erp_table)
653 switch (erp_table->num_atcam_erps) {
655 /* Keep using the eRP table, but correctly set the
656 * operations pointer so that when an A-TCAM eRP is
657 * deleted we will transition to use the master mask
659 erp_table->ops = &erp_two_masks_ops;
662 /* We only kept the eRP table because we had C-TCAM
663 * eRPs in use. Now that the last C-TCAM eRP is gone we
664 * can stop using the table and transition to use the
667 mlxsw_sp_acl_erp_region_master_mask_trans(erp_table);
668 erp_table->ops = &erp_single_mask_ops;
671 /* There are no more eRPs of any kind used by the region
672 * so free its eRP table and transition to initial state
674 mlxsw_sp_acl_erp_table_disable(erp_table);
675 mlxsw_sp_acl_erp_table_free(erp_table->erp_core,
676 erp_table->num_max_atcam_erps,
677 erp_table->aregion->type,
678 erp_table->base_index);
679 erp_table->ops = &erp_no_mask_ops;
686 static struct mlxsw_sp_acl_erp *
687 __mlxsw_sp_acl_erp_ctcam_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
688 struct mlxsw_sp_acl_erp_key *key)
690 struct mlxsw_sp_acl_erp *erp;
693 erp = kzalloc(sizeof(*erp), GFP_KERNEL);
695 return ERR_PTR(-ENOMEM);
697 memcpy(&erp->key, key, sizeof(*key));
698 bitmap_from_arr32(erp->mask_bitmap, (u32 *) key->mask,
699 MLXSW_SP_ACL_TCAM_MASK_LEN);
700 refcount_set(&erp->refcnt, 1);
701 erp_table->num_ctcam_erps++;
702 erp->erp_table = erp_table;
704 err = mlxsw_sp_acl_erp_master_mask_set(erp_table, erp);
706 goto err_master_mask_set;
708 err = rhashtable_insert_fast(&erp_table->erp_ht, &erp->ht_node,
709 mlxsw_sp_acl_erp_ht_params);
711 goto err_rhashtable_insert;
713 err = mlxsw_sp_acl_erp_region_ctcam_enable(erp_table);
715 goto err_erp_region_ctcam_enable;
717 /* When C-TCAM is used, the eRP table must be used */
718 erp_table->ops = &erp_multiple_masks_ops;
722 err_erp_region_ctcam_enable:
723 rhashtable_remove_fast(&erp_table->erp_ht, &erp->ht_node,
724 mlxsw_sp_acl_erp_ht_params);
725 err_rhashtable_insert:
726 mlxsw_sp_acl_erp_master_mask_clear(erp_table, erp);
728 erp_table->num_ctcam_erps--;
733 static struct mlxsw_sp_acl_erp *
734 mlxsw_sp_acl_erp_ctcam_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
735 struct mlxsw_sp_acl_erp_key *key)
737 struct mlxsw_sp_acl_erp *erp;
740 /* There is a special situation where we need to spill rules
741 * into the C-TCAM, yet the region is still using a master
742 * mask and thus not performing a lookup in the C-TCAM. This
743 * can happen when two rules that only differ in priority - and
744 * thus sharing the same key - are programmed. In this case
745 * we transition the region to use an eRP table
747 err = mlxsw_sp_acl_erp_region_table_trans(erp_table);
751 erp = __mlxsw_sp_acl_erp_ctcam_mask_create(erp_table, key);
760 mlxsw_sp_acl_erp_region_master_mask_trans(erp_table);
765 mlxsw_sp_acl_erp_ctcam_mask_destroy(struct mlxsw_sp_acl_erp *erp)
767 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table;
769 mlxsw_sp_acl_erp_region_ctcam_disable(erp_table);
770 rhashtable_remove_fast(&erp_table->erp_ht, &erp->ht_node,
771 mlxsw_sp_acl_erp_ht_params);
772 mlxsw_sp_acl_erp_master_mask_clear(erp_table, erp);
773 erp_table->num_ctcam_erps--;
776 /* Once the last C-TCAM eRP was destroyed, the state we
777 * transition to depends on the number of A-TCAM eRPs currently
780 if (erp_table->num_ctcam_erps > 0)
782 mlxsw_sp_acl_erp_ctcam_table_ops_set(erp_table);
785 static struct mlxsw_sp_acl_erp *
786 mlxsw_sp_acl_erp_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
787 struct mlxsw_sp_acl_erp_key *key)
789 struct mlxsw_sp_acl_erp *erp;
793 return __mlxsw_sp_acl_erp_ctcam_mask_create(erp_table, key);
795 /* Expand the eRP table for the new eRP, if needed */
796 err = mlxsw_sp_acl_erp_table_expand(erp_table);
800 erp = mlxsw_sp_acl_erp_generic_create(erp_table, key);
804 err = mlxsw_sp_acl_erp_index_get(erp_table, &erp->index);
806 goto err_erp_index_get;
808 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, erp);
810 goto err_table_erp_add;
812 err = mlxsw_sp_acl_erp_region_erp_add(erp_table, erp);
814 goto err_region_erp_add;
816 erp_table->ops = &erp_multiple_masks_ops;
821 mlxsw_sp_acl_erp_table_erp_del(erp);
823 mlxsw_sp_acl_erp_index_put(erp_table, erp->index);
825 mlxsw_sp_acl_erp_generic_destroy(erp);
830 mlxsw_sp_acl_erp_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
831 struct mlxsw_sp_acl_erp *erp)
834 return mlxsw_sp_acl_erp_ctcam_mask_destroy(erp);
836 mlxsw_sp_acl_erp_region_erp_del(erp);
837 mlxsw_sp_acl_erp_table_erp_del(erp);
838 mlxsw_sp_acl_erp_index_put(erp_table, erp->index);
839 mlxsw_sp_acl_erp_generic_destroy(erp);
841 if (erp_table->num_atcam_erps == 2 && erp_table->num_ctcam_erps == 0)
842 erp_table->ops = &erp_two_masks_ops;
845 static struct mlxsw_sp_acl_erp *
846 mlxsw_sp_acl_erp_second_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
847 struct mlxsw_sp_acl_erp_key *key)
849 struct mlxsw_sp_acl_erp *erp;
853 return mlxsw_sp_acl_erp_ctcam_mask_create(erp_table, key);
855 /* Transition to use eRP table instead of master mask */
856 err = mlxsw_sp_acl_erp_region_table_trans(erp_table);
860 erp = mlxsw_sp_acl_erp_generic_create(erp_table, key);
866 err = mlxsw_sp_acl_erp_index_get(erp_table, &erp->index);
868 goto err_erp_index_get;
870 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, erp);
872 goto err_table_erp_add;
874 err = mlxsw_sp_acl_erp_region_erp_add(erp_table, erp);
876 goto err_region_erp_add;
878 erp_table->ops = &erp_two_masks_ops;
883 mlxsw_sp_acl_erp_table_erp_del(erp);
885 mlxsw_sp_acl_erp_index_put(erp_table, erp->index);
887 mlxsw_sp_acl_erp_generic_destroy(erp);
889 mlxsw_sp_acl_erp_region_master_mask_trans(erp_table);
894 mlxsw_sp_acl_erp_second_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
895 struct mlxsw_sp_acl_erp *erp)
898 return mlxsw_sp_acl_erp_ctcam_mask_destroy(erp);
900 mlxsw_sp_acl_erp_region_erp_del(erp);
901 mlxsw_sp_acl_erp_table_erp_del(erp);
902 mlxsw_sp_acl_erp_index_put(erp_table, erp->index);
903 mlxsw_sp_acl_erp_generic_destroy(erp);
904 /* Transition to use master mask instead of eRP table */
905 mlxsw_sp_acl_erp_region_master_mask_trans(erp_table);
907 erp_table->ops = &erp_single_mask_ops;
910 static struct mlxsw_sp_acl_erp *
911 mlxsw_sp_acl_erp_first_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
912 struct mlxsw_sp_acl_erp_key *key)
914 struct mlxsw_sp_acl_erp *erp;
917 return ERR_PTR(-EINVAL);
919 erp = mlxsw_sp_acl_erp_generic_create(erp_table, key);
923 erp_table->ops = &erp_single_mask_ops;
929 mlxsw_sp_acl_erp_first_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
930 struct mlxsw_sp_acl_erp *erp)
932 mlxsw_sp_acl_erp_generic_destroy(erp);
933 erp_table->ops = &erp_no_mask_ops;
937 mlxsw_sp_acl_erp_no_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
938 struct mlxsw_sp_acl_erp *erp)
943 struct mlxsw_sp_acl_erp *
944 mlxsw_sp_acl_erp_get(struct mlxsw_sp_acl_atcam_region *aregion,
945 const char *mask, bool ctcam)
947 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
948 struct mlxsw_sp_acl_erp_key key;
949 struct mlxsw_sp_acl_erp *erp;
951 /* eRPs are allocated from a shared resource, but currently all
952 * allocations are done under RTNL.
956 memcpy(key.mask, mask, MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN);
958 erp = rhashtable_lookup_fast(&erp_table->erp_ht, &key,
959 mlxsw_sp_acl_erp_ht_params);
961 refcount_inc(&erp->refcnt);
965 return erp_table->ops->erp_create(erp_table, &key);
968 void mlxsw_sp_acl_erp_put(struct mlxsw_sp_acl_atcam_region *aregion,
969 struct mlxsw_sp_acl_erp *erp)
971 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
975 if (!refcount_dec_and_test(&erp->refcnt))
978 erp_table->ops->erp_destroy(erp_table, erp);
981 static struct mlxsw_sp_acl_erp_table *
982 mlxsw_sp_acl_erp_table_create(struct mlxsw_sp_acl_atcam_region *aregion)
984 struct mlxsw_sp_acl_erp_table *erp_table;
987 erp_table = kzalloc(sizeof(*erp_table), GFP_KERNEL);
989 return ERR_PTR(-ENOMEM);
991 err = rhashtable_init(&erp_table->erp_ht, &mlxsw_sp_acl_erp_ht_params);
993 goto err_rhashtable_init;
995 erp_table->erp_core = aregion->atcam->erp_core;
996 erp_table->ops = &erp_no_mask_ops;
997 INIT_LIST_HEAD(&erp_table->atcam_erps_list);
998 erp_table->aregion = aregion;
1002 err_rhashtable_init:
1004 return ERR_PTR(err);
1008 mlxsw_sp_acl_erp_table_destroy(struct mlxsw_sp_acl_erp_table *erp_table)
1010 WARN_ON(!list_empty(&erp_table->atcam_erps_list));
1011 rhashtable_destroy(&erp_table->erp_ht);
1016 mlxsw_sp_acl_erp_master_mask_init(struct mlxsw_sp_acl_atcam_region *aregion)
1018 struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp;
1019 char percr_pl[MLXSW_REG_PERCR_LEN];
1021 mlxsw_reg_percr_pack(percr_pl, aregion->region->id);
1022 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(percr), percr_pl);
1026 mlxsw_sp_acl_erp_region_param_init(struct mlxsw_sp_acl_atcam_region *aregion)
1028 struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp;
1029 char pererp_pl[MLXSW_REG_PERERP_LEN];
1031 mlxsw_reg_pererp_pack(pererp_pl, aregion->region->id, false, false, 0,
1033 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl);
1036 int mlxsw_sp_acl_erp_region_init(struct mlxsw_sp_acl_atcam_region *aregion)
1038 struct mlxsw_sp_acl_erp_table *erp_table;
1041 erp_table = mlxsw_sp_acl_erp_table_create(aregion);
1042 if (IS_ERR(erp_table))
1043 return PTR_ERR(erp_table);
1044 aregion->erp_table = erp_table;
1046 /* Initialize the region's master mask to all zeroes */
1047 err = mlxsw_sp_acl_erp_master_mask_init(aregion);
1049 goto err_erp_master_mask_init;
1051 /* Initialize the region to not use the eRP table */
1052 err = mlxsw_sp_acl_erp_region_param_init(aregion);
1054 goto err_erp_region_param_init;
1058 err_erp_region_param_init:
1059 err_erp_master_mask_init:
1060 mlxsw_sp_acl_erp_table_destroy(erp_table);
1064 void mlxsw_sp_acl_erp_region_fini(struct mlxsw_sp_acl_atcam_region *aregion)
1066 mlxsw_sp_acl_erp_table_destroy(aregion->erp_table);
1070 mlxsw_sp_acl_erp_tables_sizes_query(struct mlxsw_sp *mlxsw_sp,
1071 struct mlxsw_sp_acl_erp_core *erp_core)
1075 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_2KB) ||
1076 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_4KB) ||
1077 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_8KB) ||
1078 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_12KB))
1081 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_2KB);
1082 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_2KB] = size;
1084 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_4KB);
1085 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_4KB] = size;
1087 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_8KB);
1088 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_8KB] = size;
1090 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_12KB);
1091 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_12KB] = size;
1096 static int mlxsw_sp_acl_erp_tables_init(struct mlxsw_sp *mlxsw_sp,
1097 struct mlxsw_sp_acl_erp_core *erp_core)
1099 unsigned int erpt_bank_size;
1102 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_MAX_ERPT_BANK_SIZE) ||
1103 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_MAX_ERPT_BANKS))
1105 erpt_bank_size = MLXSW_CORE_RES_GET(mlxsw_sp->core,
1106 ACL_MAX_ERPT_BANK_SIZE);
1107 erp_core->num_erp_banks = MLXSW_CORE_RES_GET(mlxsw_sp->core,
1108 ACL_MAX_ERPT_BANKS);
1110 erp_core->erp_tables = gen_pool_create(0, -1);
1111 if (!erp_core->erp_tables)
1113 gen_pool_set_algo(erp_core->erp_tables, gen_pool_best_fit, NULL);
1115 err = gen_pool_add(erp_core->erp_tables,
1116 MLXSW_SP_ACL_ERP_GENALLOC_OFFSET, erpt_bank_size,
1119 goto err_gen_pool_add;
1121 /* Different regions require masks of different sizes */
1122 err = mlxsw_sp_acl_erp_tables_sizes_query(mlxsw_sp, erp_core);
1124 goto err_erp_tables_sizes_query;
1128 err_erp_tables_sizes_query:
1130 gen_pool_destroy(erp_core->erp_tables);
1134 static void mlxsw_sp_acl_erp_tables_fini(struct mlxsw_sp *mlxsw_sp,
1135 struct mlxsw_sp_acl_erp_core *erp_core)
1137 gen_pool_destroy(erp_core->erp_tables);
1140 int mlxsw_sp_acl_erps_init(struct mlxsw_sp *mlxsw_sp,
1141 struct mlxsw_sp_acl_atcam *atcam)
1143 struct mlxsw_sp_acl_erp_core *erp_core;
1146 erp_core = kzalloc(sizeof(*erp_core), GFP_KERNEL);
1149 erp_core->mlxsw_sp = mlxsw_sp;
1150 atcam->erp_core = erp_core;
1152 err = mlxsw_sp_acl_erp_tables_init(mlxsw_sp, erp_core);
1154 goto err_erp_tables_init;
1158 err_erp_tables_init:
1163 void mlxsw_sp_acl_erps_fini(struct mlxsw_sp *mlxsw_sp,
1164 struct mlxsw_sp_acl_atcam *atcam)
1166 mlxsw_sp_acl_erp_tables_fini(mlxsw_sp, atcam->erp_core);
1167 kfree(atcam->erp_core);