DPDK  22.11.2
rte_swx_pipeline_internal.h
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2021 Intel Corporation
3  */
4 #ifndef __INCLUDE_RTE_SWX_PIPELINE_INTERNAL_H__
5 #define __INCLUDE_RTE_SWX_PIPELINE_INTERNAL_H__
6 
7 #include <inttypes.h>
8 #include <string.h>
9 #include <sys/queue.h>
10 
11 #include <rte_byteorder.h>
12 #include <rte_common.h>
13 #include <rte_cycles.h>
14 #include <rte_prefetch.h>
15 #include <rte_meter.h>
16 
17 #include <rte_swx_table_selector.h>
18 #include <rte_swx_table_learner.h>
19 #include <rte_swx_pipeline.h>
20 #include <rte_swx_ctl.h>
21 
22 #ifndef TRACE_LEVEL
23 #define TRACE_LEVEL 0
24 #endif
25 
26 #if TRACE_LEVEL
27 #define TRACE(...) printf(__VA_ARGS__)
28 #else
29 #define TRACE(...)
30 #endif
31 
32 /*
33  * Environment.
34  */
35 #define ntoh64(x) rte_be_to_cpu_64(x)
36 #define hton64(x) rte_cpu_to_be_64(x)
37 
38 /*
39  * Struct.
40  */
41 struct field {
42  char name[RTE_SWX_NAME_SIZE];
43  uint32_t n_bits;
44  uint32_t offset;
45  int var_size;
46 };
47 
48 struct struct_type {
49  TAILQ_ENTRY(struct_type) node;
50  char name[RTE_SWX_NAME_SIZE];
51  struct field *fields;
52  uint32_t n_fields;
53  uint32_t n_bits;
54  uint32_t n_bits_min;
55  int var_size;
56 };
57 
58 TAILQ_HEAD(struct_type_tailq, struct_type);
59 
60 /*
61  * Input port.
62  */
63 struct port_in_type {
64  TAILQ_ENTRY(port_in_type) node;
65  char name[RTE_SWX_NAME_SIZE];
66  struct rte_swx_port_in_ops ops;
67 };
68 
69 TAILQ_HEAD(port_in_type_tailq, port_in_type);
70 
71 struct port_in {
72  TAILQ_ENTRY(port_in) node;
73  struct port_in_type *type;
74  void *obj;
75  uint32_t id;
76 };
77 
78 TAILQ_HEAD(port_in_tailq, port_in);
79 
80 struct port_in_runtime {
82  void *obj;
83 };
84 
85 /*
86  * Output port.
87  */
88 struct port_out_type {
89  TAILQ_ENTRY(port_out_type) node;
90  char name[RTE_SWX_NAME_SIZE];
91  struct rte_swx_port_out_ops ops;
92 };
93 
94 TAILQ_HEAD(port_out_type_tailq, port_out_type);
95 
96 struct port_out {
97  TAILQ_ENTRY(port_out) node;
98  struct port_out_type *type;
99  void *obj;
100  uint32_t id;
101 };
102 
103 TAILQ_HEAD(port_out_tailq, port_out);
104 
105 struct port_out_runtime {
107  rte_swx_port_out_pkt_fast_clone_tx_t pkt_fast_clone_tx;
108  rte_swx_port_out_pkt_clone_tx_t pkt_clone_tx;
110  void *obj;
111 };
112 
113 /*
114  * Packet mirroring.
115  */
116 struct mirroring_session {
117  uint32_t port_id;
118  int fast_clone;
119  uint32_t truncation_length;
120 };
121 
122 /*
123  * Extern object.
124  */
125 struct extern_type_member_func {
126  TAILQ_ENTRY(extern_type_member_func) node;
127  char name[RTE_SWX_NAME_SIZE];
129  uint32_t id;
130 };
131 
132 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
133 
134 struct extern_type {
135  TAILQ_ENTRY(extern_type) node;
136  char name[RTE_SWX_NAME_SIZE];
137  struct struct_type *mailbox_struct_type;
140  struct extern_type_member_func_tailq funcs;
141  uint32_t n_funcs;
142 };
143 
144 TAILQ_HEAD(extern_type_tailq, extern_type);
145 
146 struct extern_obj {
147  TAILQ_ENTRY(extern_obj) node;
148  char name[RTE_SWX_NAME_SIZE];
149  struct extern_type *type;
150  void *obj;
151  uint32_t struct_id;
152  uint32_t id;
153 };
154 
155 TAILQ_HEAD(extern_obj_tailq, extern_obj);
156 
157 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
158 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
159 #endif
160 
161 struct extern_obj_runtime {
162  void *obj;
163  uint8_t *mailbox;
164  rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
165 };
166 
167 /*
168  * Extern function.
169  */
170 struct extern_func {
171  TAILQ_ENTRY(extern_func) node;
172  char name[RTE_SWX_NAME_SIZE];
173  struct struct_type *mailbox_struct_type;
175  uint32_t struct_id;
176  uint32_t id;
177 };
178 
179 TAILQ_HEAD(extern_func_tailq, extern_func);
180 
181 struct extern_func_runtime {
182  uint8_t *mailbox;
184 };
185 
186 /*
187  * Hash function.
188  */
189 struct hash_func {
190  TAILQ_ENTRY(hash_func) node;
191  char name[RTE_SWX_NAME_SIZE];
192  rte_swx_hash_func_t func;
193  uint32_t id;
194 };
195 
196 TAILQ_HEAD(hash_func_tailq, hash_func);
197 
198 struct hash_func_runtime {
199  rte_swx_hash_func_t func;
200 };
201 
202 /*
203  * Header.
204  */
205 struct header {
206  TAILQ_ENTRY(header) node;
207  char name[RTE_SWX_NAME_SIZE];
208  struct struct_type *st;
209  uint32_t struct_id;
210  uint32_t id;
211 };
212 
213 TAILQ_HEAD(header_tailq, header);
214 
215 struct header_runtime {
216  uint8_t *ptr0;
217  uint32_t n_bytes;
218 };
219 
220 struct header_out_runtime {
221  uint8_t *ptr0;
222  uint8_t *ptr;
223  uint32_t n_bytes;
224 };
225 
226 /*
227  * Instruction.
228  */
229 
230 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
231  * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
232  * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
233  * when transferred to packet meta-data and in NBO when transferred to packet
234  * headers.
235  */
236 
237 /* Notation conventions:
238  * -Header field: H = h.header.field (dst/src)
239  * -Meta-data field: M = m.field (dst/src)
240  * -Extern object mailbox field: E = e.field (dst/src)
241  * -Extern function mailbox field: F = f.field (dst/src)
242  * -Table action data field: T = t.field (src only)
243  * -Immediate value: I = 32-bit unsigned value (src only)
244  */
245 
246 enum instruction_type {
247  /* rx m.port_in */
248  INSTR_RX,
249 
250  /* tx port_out
251  * port_out = MI
252  */
253  INSTR_TX, /* port_out = M */
254  INSTR_TX_I, /* port_out = I */
255  INSTR_DROP,
256 
257  /*
258  * mirror slot_id session_id
259  * slot_id = MEFT
260  * session_id = MEFT
261  */
262  INSTR_MIRROR,
263 
264  /* recirculate
265  */
266  INSTR_RECIRCULATE,
267 
268  /* recircid m.recirc_pass_id
269  * Read the internal recirculation pass ID into the specified meta-data field.
270  */
271  INSTR_RECIRCID,
272 
273  /* extract h.header */
274  INSTR_HDR_EXTRACT,
275  INSTR_HDR_EXTRACT2,
276  INSTR_HDR_EXTRACT3,
277  INSTR_HDR_EXTRACT4,
278  INSTR_HDR_EXTRACT5,
279  INSTR_HDR_EXTRACT6,
280  INSTR_HDR_EXTRACT7,
281  INSTR_HDR_EXTRACT8,
282 
283  /* extract h.header m.last_field_size */
284  INSTR_HDR_EXTRACT_M,
285 
286  /* lookahead h.header */
287  INSTR_HDR_LOOKAHEAD,
288 
289  /* emit h.header */
290  INSTR_HDR_EMIT,
291  INSTR_HDR_EMIT_TX,
292  INSTR_HDR_EMIT2_TX,
293  INSTR_HDR_EMIT3_TX,
294  INSTR_HDR_EMIT4_TX,
295  INSTR_HDR_EMIT5_TX,
296  INSTR_HDR_EMIT6_TX,
297  INSTR_HDR_EMIT7_TX,
298  INSTR_HDR_EMIT8_TX,
299 
300  /* validate h.header */
301  INSTR_HDR_VALIDATE,
302 
303  /* invalidate h.header */
304  INSTR_HDR_INVALIDATE,
305 
306  /* mov dst src
307  * dst = src
308  * dst = HMEF, src = HMEFTI
309  */
310  INSTR_MOV, /* dst = MEF, src = MEFT; size(dst) <= 64 bits, size(src) <= 64 bits. */
311  INSTR_MOV_MH, /* dst = MEF, src = H; size(dst) <= 64 bits, size(src) <= 64 bits. */
312  INSTR_MOV_HM, /* dst = H, src = MEFT; size(dst) <= 64 bits, size(src) <= 64 bits. */
313  INSTR_MOV_HH, /* dst = H, src = H; size(dst) <= 64 bits, size(src) <= 64 bits. */
314  INSTR_MOV_DMA, /* dst = HMEF, src = HMEF; size(dst) = size(src) > 64 bits, NBO format. */
315  INSTR_MOV_128, /* dst = HMEF, src = HMEF; size(dst) = size(src) = 128 bits, NBO format. */
316  INSTR_MOV_I, /* dst = HMEF, src = I; size(dst) <= 64 bits. */
317 
318  /* dma h.header t.field
319  * memcpy(h.header, t.field, sizeof(h.header))
320  */
321  INSTR_DMA_HT,
322  INSTR_DMA_HT2,
323  INSTR_DMA_HT3,
324  INSTR_DMA_HT4,
325  INSTR_DMA_HT5,
326  INSTR_DMA_HT6,
327  INSTR_DMA_HT7,
328  INSTR_DMA_HT8,
329 
330  /* add dst src
331  * dst += src
332  * dst = HMEF, src = HMEFTI
333  */
334  INSTR_ALU_ADD, /* dst = MEF, src = MEF */
335  INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
336  INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
337  INSTR_ALU_ADD_HH, /* dst = H, src = H */
338  INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
339  INSTR_ALU_ADD_HI, /* dst = H, src = I */
340 
341  /* sub dst src
342  * dst -= src
343  * dst = HMEF, src = HMEFTI
344  */
345  INSTR_ALU_SUB, /* dst = MEF, src = MEF */
346  INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
347  INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
348  INSTR_ALU_SUB_HH, /* dst = H, src = H */
349  INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
350  INSTR_ALU_SUB_HI, /* dst = H, src = I */
351 
352  /* ckadd dst src
353  * dst = dst '+ src[0:1] '+ src[2:3] '+ ...
354  * dst = H, src = {H, h.header}, '+ = 1's complement addition operator
355  */
356  INSTR_ALU_CKADD_FIELD, /* src = H */
357  INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 bytes. */
358  INSTR_ALU_CKADD_STRUCT, /* src = h.header, with sizeof(header) any 4-byte multiple. */
359 
360  /* cksub dst src
361  * dst = dst '- src
362  * dst = H, src = H, '- = 1's complement subtraction operator
363  */
364  INSTR_ALU_CKSUB_FIELD,
365 
366  /* and dst src
367  * dst &= src
368  * dst = HMEF, src = HMEFTI
369  */
370  INSTR_ALU_AND, /* dst = MEF, src = MEFT */
371  INSTR_ALU_AND_MH, /* dst = MEF, src = H */
372  INSTR_ALU_AND_HM, /* dst = H, src = MEFT */
373  INSTR_ALU_AND_HH, /* dst = H, src = H */
374  INSTR_ALU_AND_I, /* dst = HMEF, src = I */
375 
376  /* or dst src
377  * dst |= src
378  * dst = HMEF, src = HMEFTI
379  */
380  INSTR_ALU_OR, /* dst = MEF, src = MEFT */
381  INSTR_ALU_OR_MH, /* dst = MEF, src = H */
382  INSTR_ALU_OR_HM, /* dst = H, src = MEFT */
383  INSTR_ALU_OR_HH, /* dst = H, src = H */
384  INSTR_ALU_OR_I, /* dst = HMEF, src = I */
385 
386  /* xor dst src
387  * dst ^= src
388  * dst = HMEF, src = HMEFTI
389  */
390  INSTR_ALU_XOR, /* dst = MEF, src = MEFT */
391  INSTR_ALU_XOR_MH, /* dst = MEF, src = H */
392  INSTR_ALU_XOR_HM, /* dst = H, src = MEFT */
393  INSTR_ALU_XOR_HH, /* dst = H, src = H */
394  INSTR_ALU_XOR_I, /* dst = HMEF, src = I */
395 
396  /* shl dst src
397  * dst <<= src
398  * dst = HMEF, src = HMEFTI
399  */
400  INSTR_ALU_SHL, /* dst = MEF, src = MEF */
401  INSTR_ALU_SHL_MH, /* dst = MEF, src = H */
402  INSTR_ALU_SHL_HM, /* dst = H, src = MEF */
403  INSTR_ALU_SHL_HH, /* dst = H, src = H */
404  INSTR_ALU_SHL_MI, /* dst = MEF, src = I */
405  INSTR_ALU_SHL_HI, /* dst = H, src = I */
406 
407  /* shr dst src
408  * dst >>= src
409  * dst = HMEF, src = HMEFTI
410  */
411  INSTR_ALU_SHR, /* dst = MEF, src = MEF */
412  INSTR_ALU_SHR_MH, /* dst = MEF, src = H */
413  INSTR_ALU_SHR_HM, /* dst = H, src = MEF */
414  INSTR_ALU_SHR_HH, /* dst = H, src = H */
415  INSTR_ALU_SHR_MI, /* dst = MEF, src = I */
416  INSTR_ALU_SHR_HI, /* dst = H, src = I */
417 
418  /* regprefetch REGARRAY index
419  * prefetch REGARRAY[index]
420  * index = HMEFTI
421  */
422  INSTR_REGPREFETCH_RH, /* index = H */
423  INSTR_REGPREFETCH_RM, /* index = MEFT */
424  INSTR_REGPREFETCH_RI, /* index = I */
425 
426  /* regrd dst REGARRAY index
427  * dst = REGARRAY[index]
428  * dst = HMEF, index = HMEFTI
429  */
430  INSTR_REGRD_HRH, /* dst = H, index = H */
431  INSTR_REGRD_HRM, /* dst = H, index = MEFT */
432  INSTR_REGRD_HRI, /* dst = H, index = I */
433  INSTR_REGRD_MRH, /* dst = MEF, index = H */
434  INSTR_REGRD_MRM, /* dst = MEF, index = MEFT */
435  INSTR_REGRD_MRI, /* dst = MEF, index = I */
436 
437  /* regwr REGARRAY index src
438  * REGARRAY[index] = src
439  * index = HMEFTI, src = HMEFTI
440  */
441  INSTR_REGWR_RHH, /* index = H, src = H */
442  INSTR_REGWR_RHM, /* index = H, src = MEFT */
443  INSTR_REGWR_RHI, /* index = H, src = I */
444  INSTR_REGWR_RMH, /* index = MEFT, src = H */
445  INSTR_REGWR_RMM, /* index = MEFT, src = MEFT */
446  INSTR_REGWR_RMI, /* index = MEFT, src = I */
447  INSTR_REGWR_RIH, /* index = I, src = H */
448  INSTR_REGWR_RIM, /* index = I, src = MEFT */
449  INSTR_REGWR_RII, /* index = I, src = I */
450 
451  /* regadd REGARRAY index src
452  * REGARRAY[index] += src
453  * index = HMEFTI, src = HMEFTI
454  */
455  INSTR_REGADD_RHH, /* index = H, src = H */
456  INSTR_REGADD_RHM, /* index = H, src = MEFT */
457  INSTR_REGADD_RHI, /* index = H, src = I */
458  INSTR_REGADD_RMH, /* index = MEFT, src = H */
459  INSTR_REGADD_RMM, /* index = MEFT, src = MEFT */
460  INSTR_REGADD_RMI, /* index = MEFT, src = I */
461  INSTR_REGADD_RIH, /* index = I, src = H */
462  INSTR_REGADD_RIM, /* index = I, src = MEFT */
463  INSTR_REGADD_RII, /* index = I, src = I */
464 
465  /* metprefetch METARRAY index
466  * prefetch METARRAY[index]
467  * index = HMEFTI
468  */
469  INSTR_METPREFETCH_H, /* index = H */
470  INSTR_METPREFETCH_M, /* index = MEFT */
471  INSTR_METPREFETCH_I, /* index = I */
472 
473  /* meter METARRAY index length color_in color_out
474  * color_out = meter(METARRAY[index], length, color_in)
475  * index = HMEFTI, length = HMEFT, color_in = MEFTI, color_out = MEF
476  */
477  INSTR_METER_HHM, /* index = H, length = H, color_in = MEFT */
478  INSTR_METER_HHI, /* index = H, length = H, color_in = I */
479  INSTR_METER_HMM, /* index = H, length = MEFT, color_in = MEFT */
480  INSTR_METER_HMI, /* index = H, length = MEFT, color_in = I */
481  INSTR_METER_MHM, /* index = MEFT, length = H, color_in = MEFT */
482  INSTR_METER_MHI, /* index = MEFT, length = H, color_in = I */
483  INSTR_METER_MMM, /* index = MEFT, length = MEFT, color_in = MEFT */
484  INSTR_METER_MMI, /* index = MEFT, length = MEFT, color_in = I */
485  INSTR_METER_IHM, /* index = I, length = H, color_in = MEFT */
486  INSTR_METER_IHI, /* index = I, length = H, color_in = I */
487  INSTR_METER_IMM, /* index = I, length = MEFT, color_in = MEFT */
488  INSTR_METER_IMI, /* index = I, length = MEFT, color_in = I */
489 
490  /* table TABLE */
491  INSTR_TABLE,
492  INSTR_TABLE_AF,
493  INSTR_SELECTOR,
494  INSTR_LEARNER,
495  INSTR_LEARNER_AF,
496 
497  /* learn ACTION_NAME [ m.action_first_arg ] m.timeout_id */
498  INSTR_LEARNER_LEARN,
499 
500  /* rearm [ m.timeout_id ] */
501  INSTR_LEARNER_REARM,
502  INSTR_LEARNER_REARM_NEW,
503 
504  /* forget */
505  INSTR_LEARNER_FORGET,
506 
507  /* entryid m.table_entry_id
508  * Read the internal table entry ID into the specified meta-data field.
509  */
510  INSTR_ENTRYID,
511 
512  /* extern e.obj.func */
513  INSTR_EXTERN_OBJ,
514 
515  /* extern f.func */
516  INSTR_EXTERN_FUNC,
517 
518  /* hash HASH_FUNC_NAME dst src_first src_last
519  * Compute hash value over range of struct fields.
520  * dst = M
521  * src_first = HMEFT
522  * src_last = HMEFT
523  * src_first and src_last must be fields within the same struct
524  */
525  INSTR_HASH_FUNC,
526 
527  /* jmp LABEL
528  * Unconditional jump
529  */
530  INSTR_JMP,
531 
532  /* jmpv LABEL h.header
533  * Jump if header is valid
534  */
535  INSTR_JMP_VALID,
536 
537  /* jmpnv LABEL h.header
538  * Jump if header is invalid
539  */
540  INSTR_JMP_INVALID,
541 
542  /* jmph LABEL
543  * Jump if table lookup hit
544  */
545  INSTR_JMP_HIT,
546 
547  /* jmpnh LABEL
548  * Jump if table lookup miss
549  */
550  INSTR_JMP_MISS,
551 
552  /* jmpa LABEL ACTION
553  * Jump if action run
554  */
555  INSTR_JMP_ACTION_HIT,
556 
557  /* jmpna LABEL ACTION
558  * Jump if action not run
559  */
560  INSTR_JMP_ACTION_MISS,
561 
562  /* jmpeq LABEL a b
563  * Jump if a is equal to b
564  * a = HMEFT, b = HMEFTI
565  */
566  INSTR_JMP_EQ, /* a = MEFT, b = MEFT */
567  INSTR_JMP_EQ_MH, /* a = MEFT, b = H */
568  INSTR_JMP_EQ_HM, /* a = H, b = MEFT */
569  INSTR_JMP_EQ_HH, /* a = H, b = H */
570  INSTR_JMP_EQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
571 
572  /* jmpneq LABEL a b
573  * Jump if a is not equal to b
574  * a = HMEFT, b = HMEFTI
575  */
576  INSTR_JMP_NEQ, /* a = MEFT, b = MEFT */
577  INSTR_JMP_NEQ_MH, /* a = MEFT, b = H */
578  INSTR_JMP_NEQ_HM, /* a = H, b = MEFT */
579  INSTR_JMP_NEQ_HH, /* a = H, b = H */
580  INSTR_JMP_NEQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
581 
582  /* jmplt LABEL a b
583  * Jump if a is less than b
584  * a = HMEFT, b = HMEFTI
585  */
586  INSTR_JMP_LT, /* a = MEFT, b = MEFT */
587  INSTR_JMP_LT_MH, /* a = MEFT, b = H */
588  INSTR_JMP_LT_HM, /* a = H, b = MEFT */
589  INSTR_JMP_LT_HH, /* a = H, b = H */
590  INSTR_JMP_LT_MI, /* a = MEFT, b = I */
591  INSTR_JMP_LT_HI, /* a = H, b = I */
592 
593  /* jmpgt LABEL a b
594  * Jump if a is greater than b
595  * a = HMEFT, b = HMEFTI
596  */
597  INSTR_JMP_GT, /* a = MEFT, b = MEFT */
598  INSTR_JMP_GT_MH, /* a = MEFT, b = H */
599  INSTR_JMP_GT_HM, /* a = H, b = MEFT */
600  INSTR_JMP_GT_HH, /* a = H, b = H */
601  INSTR_JMP_GT_MI, /* a = MEFT, b = I */
602  INSTR_JMP_GT_HI, /* a = H, b = I */
603 
604  /* return
605  * Return from action
606  */
607  INSTR_RETURN,
608 
609  /* Start of custom instructions. */
610  INSTR_CUSTOM_0,
611 };
612 
613 struct instr_operand {
614  uint8_t struct_id;
615  uint8_t n_bits;
616  uint8_t offset;
617  uint8_t pad;
618 };
619 
620 struct instr_io {
621  struct {
622  union {
623  struct {
624  uint8_t offset;
625  uint8_t n_bits;
626  uint8_t pad[2];
627  };
628 
629  uint32_t val;
630  };
631  } io;
632 
633  struct {
634  uint8_t header_id[8];
635  uint8_t struct_id[8];
636  uint8_t n_bytes[8];
637  } hdr;
638 };
639 
640 struct instr_hdr_validity {
641  uint8_t header_id;
642  uint8_t struct_id;
643 };
644 
645 struct instr_table {
646  uint8_t table_id;
647 };
648 
649 struct instr_learn {
650  uint8_t action_id;
651  uint8_t mf_first_arg_offset;
652  uint8_t mf_timeout_id_offset;
653  uint8_t mf_timeout_id_n_bits;
654 };
655 
656 struct instr_extern_obj {
657  uint8_t ext_obj_id;
658  uint8_t func_id;
659 };
660 
661 struct instr_extern_func {
662  uint8_t ext_func_id;
663 };
664 
665 struct instr_hash_func {
666  uint8_t hash_func_id;
667 
668  struct {
669  uint8_t offset;
670  uint8_t n_bits;
671  } dst;
672 
673  struct {
674  uint8_t struct_id;
675  uint16_t offset;
676  uint16_t n_bytes;
677  } src;
678 };
679 
680 struct instr_dst_src {
681  struct instr_operand dst;
682  union {
683  struct instr_operand src;
684  uint64_t src_val;
685  };
686 };
687 
688 struct instr_regarray {
689  uint8_t regarray_id;
690  uint8_t pad[3];
691 
692  union {
693  struct instr_operand idx;
694  uint32_t idx_val;
695  };
696 
697  union {
698  struct instr_operand dstsrc;
699  uint64_t dstsrc_val;
700  };
701 };
702 
703 struct instr_meter {
704  uint8_t metarray_id;
705  uint8_t pad[3];
706 
707  union {
708  struct instr_operand idx;
709  uint32_t idx_val;
710  };
711 
712  struct instr_operand length;
713 
714  union {
715  struct instr_operand color_in;
716  uint32_t color_in_val;
717  };
718 
719  struct instr_operand color_out;
720 };
721 
722 struct instr_dma {
723  struct {
724  uint8_t header_id[8];
725  uint8_t struct_id[8];
726  } dst;
727 
728  struct {
729  uint8_t offset[8];
730  } src;
731 
732  uint16_t n_bytes[8];
733 };
734 
735 struct instr_jmp {
736  struct instruction *ip;
737 
738  union {
739  struct instr_operand a;
740  uint8_t header_id;
741  uint8_t action_id;
742  };
743 
744  union {
745  struct instr_operand b;
746  uint64_t b_val;
747  };
748 };
749 
750 struct instruction {
751  enum instruction_type type;
752  union {
753  struct instr_io io;
754  struct instr_dst_src mirror;
755  struct instr_hdr_validity valid;
756  struct instr_dst_src mov;
757  struct instr_regarray regarray;
758  struct instr_meter meter;
759  struct instr_dma dma;
760  struct instr_dst_src alu;
761  struct instr_table table;
762  struct instr_learn learn;
763  struct instr_extern_obj ext_obj;
764  struct instr_extern_func ext_func;
765  struct instr_hash_func hash_func;
766  struct instr_jmp jmp;
767  };
768 };
769 
770 struct instruction_data {
771  char label[RTE_SWX_NAME_SIZE];
772  char jmp_label[RTE_SWX_NAME_SIZE];
773  uint32_t n_users; /* user = jmp instruction to this instruction. */
774  int invalid;
775 };
776 
777 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
778 
779 /*
780  * Action.
781  */
782 typedef void
783 (*action_func_t)(struct rte_swx_pipeline *p);
784 
785 struct action {
786  TAILQ_ENTRY(action) node;
787  char name[RTE_SWX_NAME_SIZE];
788  struct struct_type *st;
789  int *args_endianness; /* 0 = Host Byte Order (HBO); 1 = Network Byte Order (NBO). */
790  struct instruction *instructions;
791  struct instruction_data *instruction_data;
792  uint32_t n_instructions;
793  uint32_t id;
794 };
795 
796 TAILQ_HEAD(action_tailq, action);
797 
798 /*
799  * Table.
800  */
801 struct table_type {
802  TAILQ_ENTRY(table_type) node;
803  char name[RTE_SWX_NAME_SIZE];
804  enum rte_swx_table_match_type match_type;
805  struct rte_swx_table_ops ops;
806 };
807 
808 TAILQ_HEAD(table_type_tailq, table_type);
809 
810 struct match_field {
811  enum rte_swx_table_match_type match_type;
812  struct field *field;
813 };
814 
815 struct table {
816  TAILQ_ENTRY(table) node;
817  char name[RTE_SWX_NAME_SIZE];
818  char args[RTE_SWX_NAME_SIZE];
819  struct table_type *type; /* NULL when n_fields == 0. */
820 
821  /* Match. */
822  struct match_field *fields;
823  uint32_t n_fields;
824  struct header *header; /* Only valid when n_fields > 0. */
825 
826  /* Action. */
827  struct action **actions;
828  struct action *default_action;
829  uint8_t *default_action_data;
830  uint32_t n_actions;
831  int default_action_is_const;
832  uint32_t action_data_size_max;
833  int *action_is_for_table_entries;
834  int *action_is_for_default_entry;
835 
836  struct hash_func *hf;
837  uint32_t size;
838  uint32_t id;
839 };
840 
841 TAILQ_HEAD(table_tailq, table);
842 
843 struct table_runtime {
845  void *mailbox;
846  uint8_t **key;
847 };
848 
849 struct table_statistics {
850  uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
851  uint64_t *n_pkts_action;
852 };
853 
854 /*
855  * Selector.
856  */
857 struct selector {
858  TAILQ_ENTRY(selector) node;
859  char name[RTE_SWX_NAME_SIZE];
860 
861  struct field *group_id_field;
862  struct field **selector_fields;
863  uint32_t n_selector_fields;
864  struct header *selector_header;
865  struct field *member_id_field;
866 
867  uint32_t n_groups_max;
868  uint32_t n_members_per_group_max;
869 
870  uint32_t id;
871 };
872 
873 TAILQ_HEAD(selector_tailq, selector);
874 
875 struct selector_runtime {
876  void *mailbox;
877  uint8_t **group_id_buffer;
878  uint8_t **selector_buffer;
879  uint8_t **member_id_buffer;
880 };
881 
882 struct selector_statistics {
883  uint64_t n_pkts;
884 };
885 
886 /*
887  * Learner table.
888  */
889 struct learner {
890  TAILQ_ENTRY(learner) node;
891  char name[RTE_SWX_NAME_SIZE];
892 
893  /* Match. */
894  struct field **fields;
895  uint32_t n_fields;
896  struct header *header;
897 
898  /* Action. */
899  struct action **actions;
900  struct action *default_action;
901  uint8_t *default_action_data;
902  uint32_t n_actions;
903  int default_action_is_const;
904  uint32_t action_data_size_max;
905  int *action_is_for_table_entries;
906  int *action_is_for_default_entry;
907 
908  struct hash_func *hf;
909  uint32_t size;
911  uint32_t n_timeouts;
912  uint32_t id;
913 };
914 
915 TAILQ_HEAD(learner_tailq, learner);
916 
917 struct learner_runtime {
918  void *mailbox;
919  uint8_t **key;
920 };
921 
922 struct learner_statistics {
923  uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
924  uint64_t n_pkts_learn[2]; /* 0 = Learn OK, 1 = Learn error. */
925  uint64_t n_pkts_rearm;
926  uint64_t n_pkts_forget;
927  uint64_t *n_pkts_action;
928 };
929 
930 /*
931  * Register array.
932  */
933 struct regarray {
934  TAILQ_ENTRY(regarray) node;
935  char name[RTE_SWX_NAME_SIZE];
936  uint64_t init_val;
937  uint32_t size;
938  uint32_t id;
939 };
940 
941 TAILQ_HEAD(regarray_tailq, regarray);
942 
943 struct regarray_runtime {
944  uint64_t *regarray;
945  uint32_t size_mask;
946 };
947 
948 /*
949  * Meter array.
950  */
951 struct meter_profile {
952  TAILQ_ENTRY(meter_profile) node;
953  char name[RTE_SWX_NAME_SIZE];
954  struct rte_meter_trtcm_params params;
955  struct rte_meter_trtcm_profile profile;
956  uint32_t n_users;
957 };
958 
959 TAILQ_HEAD(meter_profile_tailq, meter_profile);
960 
961 struct metarray {
962  TAILQ_ENTRY(metarray) node;
963  char name[RTE_SWX_NAME_SIZE];
964  uint32_t size;
965  uint32_t id;
966 };
967 
968 TAILQ_HEAD(metarray_tailq, metarray);
969 
970 struct meter {
971  struct rte_meter_trtcm m;
972  struct meter_profile *profile;
973  enum rte_color color_mask;
974  uint8_t pad[20];
975 
976  uint64_t n_pkts[RTE_COLORS];
977  uint64_t n_bytes[RTE_COLORS];
978 };
979 
980 struct metarray_runtime {
981  struct meter *metarray;
982  uint32_t size_mask;
983 };
984 
985 /*
986  * Pipeline.
987  */
988 struct thread {
989  /* Packet. */
990  struct rte_swx_pkt pkt;
991  uint8_t *ptr;
992  uint32_t *mirroring_slots;
993  uint64_t mirroring_slots_mask;
994  int recirculate;
995  uint32_t recirc_pass_id;
996 
997  /* Structures. */
998  uint8_t **structs;
999 
1000  /* Packet headers. */
1001  struct header_runtime *headers; /* Extracted or generated headers. */
1002  struct header_out_runtime *headers_out; /* Emitted headers. */
1003  uint8_t *header_storage;
1004  uint8_t *header_out_storage;
1005  uint64_t valid_headers;
1006  uint32_t n_headers_out;
1007 
1008  /* Packet meta-data. */
1009  uint8_t *metadata;
1010 
1011  /* Tables. */
1012  struct table_runtime *tables;
1013  struct selector_runtime *selectors;
1014  struct learner_runtime *learners;
1015  struct rte_swx_table_state *table_state;
1016  uint64_t action_id;
1017  size_t entry_id;
1018  int hit; /* 0 = Miss, 1 = Hit. */
1019  uint32_t learner_id;
1020  uint64_t time;
1021 
1022  /* Extern objects and functions. */
1023  struct extern_obj_runtime *extern_objs;
1024  struct extern_func_runtime *extern_funcs;
1025 
1026  /* Instructions. */
1027  struct instruction *ip;
1028  struct instruction *ret;
1029 };
1030 
1031 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
1032 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
1033 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
1034 
1035 #define HEADER_VALID(thread, header_id) \
1036  MASK64_BIT_GET((thread)->valid_headers, header_id)
1037 
1038 static inline uint64_t
1039 instr_operand_hbo(struct thread *t, const struct instr_operand *x)
1040 {
1041  uint8_t *x_struct = t->structs[x->struct_id];
1042  uint64_t *x64_ptr = (uint64_t *)&x_struct[x->offset];
1043  uint64_t x64 = *x64_ptr;
1044  uint64_t x64_mask = UINT64_MAX >> (64 - x->n_bits);
1045 
1046  return x64 & x64_mask;
1047 }
1048 
1049 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1050 
1051 static inline uint64_t
1052 instr_operand_nbo(struct thread *t, const struct instr_operand *x)
1053 {
1054  uint8_t *x_struct = t->structs[x->struct_id];
1055  uint64_t *x64_ptr = (uint64_t *)&x_struct[x->offset];
1056  uint64_t x64 = *x64_ptr;
1057 
1058  return ntoh64(x64) >> (64 - x->n_bits);
1059 }
1060 
1061 #else
1062 
1063 #define instr_operand_nbo instr_operand_hbo
1064 
1065 #endif
1066 
1067 #define ALU(thread, ip, operator) \
1068 { \
1069  uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1070  uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1071  uint64_t dst64 = *dst64_ptr; \
1072  uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1073  uint64_t dst = dst64 & dst64_mask; \
1074  \
1075  uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1076  uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1077  uint64_t src64 = *src64_ptr; \
1078  uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
1079  uint64_t src = src64 & src64_mask; \
1080  \
1081  uint64_t result = dst operator src; \
1082  \
1083  *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
1084 }
1085 
1086 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1087 
1088 #define ALU_MH(thread, ip, operator) \
1089 { \
1090  uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1091  uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1092  uint64_t dst64 = *dst64_ptr; \
1093  uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1094  uint64_t dst = dst64 & dst64_mask; \
1095  \
1096  uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1097  uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1098  uint64_t src64 = *src64_ptr; \
1099  uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits); \
1100  \
1101  uint64_t result = dst operator src; \
1102  \
1103  *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
1104 }
1105 
1106 #define ALU_HM(thread, ip, operator) \
1107 { \
1108  uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1109  uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1110  uint64_t dst64 = *dst64_ptr; \
1111  uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1112  uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
1113  \
1114  uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1115  uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1116  uint64_t src64 = *src64_ptr; \
1117  uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
1118  uint64_t src = src64 & src64_mask; \
1119  \
1120  uint64_t result = dst operator src; \
1121  result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
1122  \
1123  *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1124 }
1125 
1126 #define ALU_HM_FAST(thread, ip, operator) \
1127 { \
1128  uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1129  uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1130  uint64_t dst64 = *dst64_ptr; \
1131  uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1132  uint64_t dst = dst64 & dst64_mask; \
1133  \
1134  uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1135  uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1136  uint64_t src64 = *src64_ptr; \
1137  uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
1138  uint64_t src = hton64(src64 & src64_mask) >> (64 - (ip)->alu.dst.n_bits); \
1139  \
1140  uint64_t result = dst operator src; \
1141  \
1142  *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1143 }
1144 
1145 #define ALU_HH(thread, ip, operator) \
1146 { \
1147  uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1148  uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1149  uint64_t dst64 = *dst64_ptr; \
1150  uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1151  uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
1152  \
1153  uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1154  uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1155  uint64_t src64 = *src64_ptr; \
1156  uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits); \
1157  \
1158  uint64_t result = dst operator src; \
1159  result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
1160  \
1161  *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1162 }
1163 
1164 #define ALU_HH_FAST(thread, ip, operator) \
1165 { \
1166  uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1167  uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1168  uint64_t dst64 = *dst64_ptr; \
1169  uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1170  uint64_t dst = dst64 & dst64_mask; \
1171  \
1172  uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1173  uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1174  uint64_t src64 = *src64_ptr; \
1175  uint64_t src = (src64 << (64 - (ip)->alu.src.n_bits)) >> (64 - (ip)->alu.dst.n_bits); \
1176  \
1177  uint64_t result = dst operator src; \
1178  \
1179  *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1180 }
1181 
1182 #else
1183 
1184 #define ALU_MH ALU
1185 #define ALU_HM ALU
1186 #define ALU_HM_FAST ALU
1187 #define ALU_HH ALU
1188 #define ALU_HH_FAST ALU
1189 
1190 #endif
1191 
1192 #define ALU_I(thread, ip, operator) \
1193 { \
1194  uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1195  uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1196  uint64_t dst64 = *dst64_ptr; \
1197  uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1198  uint64_t dst = dst64 & dst64_mask; \
1199  \
1200  uint64_t src = (ip)->alu.src_val; \
1201  \
1202  uint64_t result = dst operator src; \
1203  \
1204  *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
1205 }
1206 
1207 #define ALU_MI ALU_I
1208 
1209 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1210 
1211 #define ALU_HI(thread, ip, operator) \
1212 { \
1213  uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1214  uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1215  uint64_t dst64 = *dst64_ptr; \
1216  uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1217  uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
1218  \
1219  uint64_t src = (ip)->alu.src_val; \
1220  \
1221  uint64_t result = dst operator src; \
1222  result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
1223  \
1224  *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1225 }
1226 
1227 #else
1228 
1229 #define ALU_HI ALU_I
1230 
1231 #endif
1232 
1233 #define MOV(thread, ip) \
1234 { \
1235  uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1236  uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1237  uint64_t dst64 = *dst64_ptr; \
1238  uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1239  \
1240  uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1241  uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1242  uint64_t src64 = *src64_ptr; \
1243  uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits); \
1244  uint64_t src = src64 & src64_mask; \
1245  \
1246  *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1247 }
1248 
1249 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1250 
1251 #define MOV_MH(thread, ip) \
1252 { \
1253  uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1254  uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1255  uint64_t dst64 = *dst64_ptr; \
1256  uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1257  \
1258  uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1259  uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1260  uint64_t src64 = *src64_ptr; \
1261  uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits); \
1262  \
1263  *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1264 }
1265 
1266 #define MOV_HM(thread, ip) \
1267 { \
1268  uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1269  uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1270  uint64_t dst64 = *dst64_ptr; \
1271  uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1272  \
1273  uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1274  uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1275  uint64_t src64 = *src64_ptr; \
1276  uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits); \
1277  uint64_t src = src64 & src64_mask; \
1278  \
1279  src = hton64(src) >> (64 - (ip)->mov.dst.n_bits); \
1280  *dst64_ptr = (dst64 & ~dst64_mask) | src; \
1281 }
1282 
1283 #define MOV_HH(thread, ip) \
1284 { \
1285  uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1286  uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1287  uint64_t dst64 = *dst64_ptr; \
1288  uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1289  \
1290  uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1291  uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1292  uint64_t src64 = *src64_ptr; \
1293  \
1294  uint64_t src = src64 << (64 - (ip)->mov.src.n_bits); \
1295  src = src >> (64 - (ip)->mov.dst.n_bits); \
1296  *dst64_ptr = (dst64 & ~dst64_mask) | src; \
1297 }
1298 
1299 #else
1300 
1301 #define MOV_MH MOV
1302 #define MOV_HM MOV
1303 #define MOV_HH MOV
1304 
1305 #endif
1306 
1307 #define MOV_I(thread, ip) \
1308 { \
1309  uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1310  uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1311  uint64_t dst64 = *dst64_ptr; \
1312  uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1313  \
1314  uint64_t src = (ip)->mov.src_val; \
1315  \
1316  *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1317 }
1318 
1319 #define JMP_CMP(thread, ip, operator) \
1320 { \
1321  uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1322  uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1323  uint64_t a64 = *a64_ptr; \
1324  uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1325  uint64_t a = a64 & a64_mask; \
1326  \
1327  uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1328  uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1329  uint64_t b64 = *b64_ptr; \
1330  uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \
1331  uint64_t b = b64 & b64_mask; \
1332  \
1333  (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1334 }
1335 
1336 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1337 
1338 #define JMP_CMP_MH(thread, ip, operator) \
1339 { \
1340  uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1341  uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1342  uint64_t a64 = *a64_ptr; \
1343  uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1344  uint64_t a = a64 & a64_mask; \
1345  \
1346  uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1347  uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1348  uint64_t b64 = *b64_ptr; \
1349  uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \
1350  \
1351  (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1352 }
1353 
1354 #define JMP_CMP_HM(thread, ip, operator) \
1355 { \
1356  uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1357  uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1358  uint64_t a64 = *a64_ptr; \
1359  uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1360  \
1361  uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1362  uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1363  uint64_t b64 = *b64_ptr; \
1364  uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \
1365  uint64_t b = b64 & b64_mask; \
1366  \
1367  (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1368 }
1369 
1370 #define JMP_CMP_HH(thread, ip, operator) \
1371 { \
1372  uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1373  uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1374  uint64_t a64 = *a64_ptr; \
1375  uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1376  \
1377  uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1378  uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1379  uint64_t b64 = *b64_ptr; \
1380  uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \
1381  \
1382  (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1383 }
1384 
1385 #define JMP_CMP_HH_FAST(thread, ip, operator) \
1386 { \
1387  uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1388  uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1389  uint64_t a64 = *a64_ptr; \
1390  uint64_t a = a64 << (64 - (ip)->jmp.a.n_bits); \
1391  \
1392  uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1393  uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1394  uint64_t b64 = *b64_ptr; \
1395  uint64_t b = b64 << (64 - (ip)->jmp.b.n_bits); \
1396  \
1397  (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1398 }
1399 
1400 #else
1401 
1402 #define JMP_CMP_MH JMP_CMP
1403 #define JMP_CMP_HM JMP_CMP
1404 #define JMP_CMP_HH JMP_CMP
1405 #define JMP_CMP_HH_FAST JMP_CMP
1406 
1407 #endif
1408 
1409 #define JMP_CMP_I(thread, ip, operator) \
1410 { \
1411  uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1412  uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1413  uint64_t a64 = *a64_ptr; \
1414  uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1415  uint64_t a = a64 & a64_mask; \
1416  \
1417  uint64_t b = (ip)->jmp.b_val; \
1418  \
1419  (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1420 }
1421 
1422 #define JMP_CMP_MI JMP_CMP_I
1423 
1424 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1425 
1426 #define JMP_CMP_HI(thread, ip, operator) \
1427 { \
1428  uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1429  uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1430  uint64_t a64 = *a64_ptr; \
1431  uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1432  \
1433  uint64_t b = (ip)->jmp.b_val; \
1434  \
1435  (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1436 }
1437 
1438 #else
1439 
1440 #define JMP_CMP_HI JMP_CMP_I
1441 
1442 #endif
1443 
1444 #define METADATA_READ(thread, offset, n_bits) \
1445 ({ \
1446  uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \
1447  uint64_t m64 = *m64_ptr; \
1448  uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits)); \
1449  (m64 & m64_mask); \
1450 })
1451 
1452 #define METADATA_WRITE(thread, offset, n_bits, value) \
1453 { \
1454  uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \
1455  uint64_t m64 = *m64_ptr; \
1456  uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits)); \
1457  \
1458  uint64_t m_new = value; \
1459  \
1460  *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask); \
1461 }
1462 
1463 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
1464 #define RTE_SWX_PIPELINE_THREADS_MAX 16
1465 #endif
1466 
1467 #ifndef RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX
1468 #define RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX 256
1469 #endif
1470 
1471 struct rte_swx_pipeline {
1472  char name[RTE_SWX_NAME_SIZE];
1473 
1474  struct struct_type_tailq struct_types;
1475  struct port_in_type_tailq port_in_types;
1476  struct port_in_tailq ports_in;
1477  struct port_out_type_tailq port_out_types;
1478  struct port_out_tailq ports_out;
1479  struct extern_type_tailq extern_types;
1480  struct extern_obj_tailq extern_objs;
1481  struct extern_func_tailq extern_funcs;
1482  struct hash_func_tailq hash_funcs;
1483  struct header_tailq headers;
1484  struct struct_type *metadata_st;
1485  uint32_t metadata_struct_id;
1486  struct action_tailq actions;
1487  struct table_type_tailq table_types;
1488  struct table_tailq tables;
1489  struct selector_tailq selectors;
1490  struct learner_tailq learners;
1491  struct regarray_tailq regarrays;
1492  struct meter_profile_tailq meter_profiles;
1493  struct metarray_tailq metarrays;
1494 
1495  struct port_in_runtime *in;
1496  struct port_out_runtime *out;
1497  struct mirroring_session *mirroring_sessions;
1498  struct instruction **action_instructions;
1499  action_func_t *action_funcs;
1500  struct rte_swx_table_state *table_state;
1501  struct table_statistics *table_stats;
1502  struct selector_statistics *selector_stats;
1503  struct learner_statistics *learner_stats;
1504  struct hash_func_runtime *hash_func_runtime;
1505  struct regarray_runtime *regarray_runtime;
1506  struct metarray_runtime *metarray_runtime;
1507  struct instruction *instructions;
1508  struct instruction_data *instruction_data;
1509  instr_exec_t *instruction_table;
1510  struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
1511  void *lib;
1512 
1513  uint32_t n_structs;
1514  uint32_t n_ports_in;
1515  uint32_t n_ports_out;
1516  uint32_t n_mirroring_slots;
1517  uint32_t n_mirroring_sessions;
1518  uint32_t n_extern_objs;
1519  uint32_t n_extern_funcs;
1520  uint32_t n_hash_funcs;
1521  uint32_t n_actions;
1522  uint32_t n_tables;
1523  uint32_t n_selectors;
1524  uint32_t n_learners;
1525  uint32_t n_regarrays;
1526  uint32_t n_metarrays;
1527  uint32_t n_headers;
1528  uint32_t thread_id;
1529  uint32_t port_id;
1530  uint32_t n_instructions;
1531  int build_done;
1532  int numa_node;
1533 };
1534 
1535 /*
1536  * Instruction.
1537  */
1538 static inline void
1539 pipeline_port_inc(struct rte_swx_pipeline *p)
1540 {
1541  p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
1542 }
1543 
1544 static inline void
1545 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
1546 {
1547  t->ip = p->instructions;
1548 }
1549 
1550 static inline void
1551 thread_ip_set(struct thread *t, struct instruction *ip)
1552 {
1553  t->ip = ip;
1554 }
1555 
1556 static inline void
1557 thread_ip_action_call(struct rte_swx_pipeline *p,
1558  struct thread *t,
1559  uint32_t action_id)
1560 {
1561  t->ret = t->ip + 1;
1562  t->ip = p->action_instructions[action_id];
1563 }
1564 
1565 static inline void
1566 thread_ip_inc(struct rte_swx_pipeline *p);
1567 
1568 static inline void
1569 thread_ip_inc(struct rte_swx_pipeline *p)
1570 {
1571  struct thread *t = &p->threads[p->thread_id];
1572 
1573  t->ip++;
1574 }
1575 
1576 static inline void
1577 thread_ip_inc_cond(struct thread *t, int cond)
1578 {
1579  t->ip += cond;
1580 }
1581 
1582 static inline void
1583 thread_yield(struct rte_swx_pipeline *p)
1584 {
1585  p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
1586 }
1587 
1588 static inline void
1589 thread_yield_cond(struct rte_swx_pipeline *p, int cond)
1590 {
1591  p->thread_id = (p->thread_id + cond) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
1592 }
1593 
1594 /*
1595  * rx.
1596  */
1597 static inline int
1598 __instr_rx_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
1599 {
1600  struct port_in_runtime *port = &p->in[p->port_id];
1601  struct rte_swx_pkt *pkt = &t->pkt;
1602  int pkt_received;
1603 
1604  /* Recirculation: keep the current packet. */
1605  if (t->recirculate) {
1606  TRACE("[Thread %2u] rx - recirculate (pass %u)\n",
1607  p->thread_id,
1608  t->recirc_pass_id + 1);
1609 
1610  /* Packet. */
1611  t->ptr = &pkt->pkt[pkt->offset];
1612  t->mirroring_slots_mask = 0;
1613  t->recirculate = 0;
1614  t->recirc_pass_id++;
1615 
1616  /* Headers. */
1617  t->valid_headers = 0;
1618  t->n_headers_out = 0;
1619 
1620  /* Tables. */
1621  t->table_state = p->table_state;
1622 
1623  return 1;
1624  }
1625 
1626  /* Packet. */
1627  pkt_received = port->pkt_rx(port->obj, pkt);
1628  t->ptr = &pkt->pkt[pkt->offset];
1629  rte_prefetch0(t->ptr);
1630 
1631  TRACE("[Thread %2u] rx %s from port %u\n",
1632  p->thread_id,
1633  pkt_received ? "1 pkt" : "0 pkts",
1634  p->port_id);
1635 
1636  t->mirroring_slots_mask = 0;
1637  t->recirc_pass_id = 0;
1638 
1639  /* Headers. */
1640  t->valid_headers = 0;
1641  t->n_headers_out = 0;
1642 
1643  /* Meta-data. */
1644  METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
1645 
1646  /* Tables. */
1647  t->table_state = p->table_state;
1648 
1649  /* Thread. */
1650  pipeline_port_inc(p);
1651 
1652  return pkt_received;
1653 }
1654 
1655 static inline void
1656 instr_rx_exec(struct rte_swx_pipeline *p)
1657 {
1658  struct thread *t = &p->threads[p->thread_id];
1659  struct instruction *ip = t->ip;
1660  int pkt_received;
1661 
1662  /* Packet. */
1663  pkt_received = __instr_rx_exec(p, t, ip);
1664 
1665  /* Thread. */
1666  thread_ip_inc_cond(t, pkt_received);
1667  thread_yield(p);
1668 }
1669 
1670 /*
1671  * tx.
1672  */
1673 static inline void
1674 emit_handler(struct thread *t)
1675 {
1676  struct header_out_runtime *h0 = &t->headers_out[0];
1677  struct header_out_runtime *h1 = &t->headers_out[1];
1678  uint32_t offset = 0, i;
1679 
1680  /* No header change or header decapsulation. */
1681  if ((t->n_headers_out == 1) &&
1682  (h0->ptr + h0->n_bytes == t->ptr)) {
1683  TRACE("Emit handler: no header change or header decap.\n");
1684 
1685  t->pkt.offset -= h0->n_bytes;
1686  t->pkt.length += h0->n_bytes;
1687 
1688  return;
1689  }
1690 
1691  /* Header encapsulation (optionally, with prior header decapsulation). */
1692  if ((t->n_headers_out == 2) &&
1693  (h1->ptr + h1->n_bytes == t->ptr) &&
1694  (h0->ptr == h0->ptr0)) {
1695  uint32_t offset;
1696 
1697  TRACE("Emit handler: header encapsulation.\n");
1698 
1699  offset = h0->n_bytes + h1->n_bytes;
1700  memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
1701  t->pkt.offset -= offset;
1702  t->pkt.length += offset;
1703 
1704  return;
1705  }
1706 
1707  /* For any other case. */
1708  TRACE("Emit handler: complex case.\n");
1709 
1710  for (i = 0; i < t->n_headers_out; i++) {
1711  struct header_out_runtime *h = &t->headers_out[i];
1712 
1713  memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
1714  offset += h->n_bytes;
1715  }
1716 
1717  if (offset) {
1718  memcpy(t->ptr - offset, t->header_out_storage, offset);
1719  t->pkt.offset -= offset;
1720  t->pkt.length += offset;
1721  }
1722 }
1723 
1724 static inline void
1725 mirroring_handler(struct rte_swx_pipeline *p, struct thread *t, struct rte_swx_pkt *pkt)
1726 {
1727  uint64_t slots_mask = t->mirroring_slots_mask, slot_mask;
1728  uint32_t slot_id;
1729 
1730  for (slot_id = 0, slot_mask = 1LLU ; slots_mask; slot_id++, slot_mask <<= 1)
1731  if (slot_mask & slots_mask) {
1732  struct port_out_runtime *port;
1733  struct mirroring_session *session;
1734  uint32_t port_id, session_id;
1735 
1736  session_id = t->mirroring_slots[slot_id];
1737  session = &p->mirroring_sessions[session_id];
1738 
1739  port_id = session->port_id;
1740  port = &p->out[port_id];
1741 
1742  if (session->fast_clone)
1743  port->pkt_fast_clone_tx(port->obj, pkt);
1744  else
1745  port->pkt_clone_tx(port->obj, pkt, session->truncation_length);
1746 
1747  slots_mask &= ~slot_mask;
1748  }
1749 }
1750 
1751 static inline void
1752 __instr_tx_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
1753 {
1754  uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
1755  struct port_out_runtime *port = &p->out[port_id];
1756  struct rte_swx_pkt *pkt = &t->pkt;
1757 
1758  /* Recirculation: keep the current packet. */
1759  if (t->recirculate) {
1760  TRACE("[Thread %2u]: tx 1 pkt - recirculate\n",
1761  p->thread_id);
1762 
1763  /* Headers. */
1764  emit_handler(t);
1765 
1766  /* Packet. */
1767  mirroring_handler(p, t, pkt);
1768 
1769  return;
1770  }
1771 
1772  TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
1773  p->thread_id,
1774  (uint32_t)port_id);
1775 
1776  /* Headers. */
1777  emit_handler(t);
1778 
1779  /* Packet. */
1780  mirroring_handler(p, t, pkt);
1781  port->pkt_tx(port->obj, pkt);
1782 }
1783 
1784 static inline void
1785 __instr_tx_i_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
1786 {
1787  uint64_t port_id = ip->io.io.val;
1788  struct port_out_runtime *port = &p->out[port_id];
1789  struct rte_swx_pkt *pkt = &t->pkt;
1790 
1791  /* Recirculation: keep the current packet. */
1792  if (t->recirculate) {
1793  TRACE("[Thread %2u]: tx (i) 1 pkt - recirculate\n",
1794  p->thread_id);
1795 
1796  /* Headers. */
1797  emit_handler(t);
1798 
1799  /* Packet. */
1800  mirroring_handler(p, t, pkt);
1801 
1802  return;
1803  }
1804 
1805  TRACE("[Thread %2u]: tx (i) 1 pkt to port %u\n",
1806  p->thread_id,
1807  (uint32_t)port_id);
1808 
1809  /* Headers. */
1810  emit_handler(t);
1811 
1812  /* Packet. */
1813  mirroring_handler(p, t, pkt);
1814  port->pkt_tx(port->obj, pkt);
1815 }
1816 
1817 static inline void
1818 __instr_drop_exec(struct rte_swx_pipeline *p,
1819  struct thread *t,
1820  const struct instruction *ip __rte_unused)
1821 {
1822  uint64_t port_id = p->n_ports_out - 1;
1823  struct port_out_runtime *port = &p->out[port_id];
1824  struct rte_swx_pkt *pkt = &t->pkt;
1825 
1826  TRACE("[Thread %2u]: drop 1 pkt\n",
1827  p->thread_id);
1828 
1829  /* Headers. */
1830  emit_handler(t);
1831 
1832  /* Packet. */
1833  mirroring_handler(p, t, pkt);
1834  port->pkt_tx(port->obj, pkt);
1835 }
1836 
1837 static inline void
1838 __instr_mirror_exec(struct rte_swx_pipeline *p,
1839  struct thread *t,
1840  const struct instruction *ip)
1841 {
1842  uint64_t slot_id = instr_operand_hbo(t, &ip->mirror.dst);
1843  uint64_t session_id = instr_operand_hbo(t, &ip->mirror.src);
1844 
1845  slot_id &= p->n_mirroring_slots - 1;
1846  session_id &= p->n_mirroring_sessions - 1;
1847 
1848  TRACE("[Thread %2u]: mirror pkt (slot = %u, session = %u)\n",
1849  p->thread_id,
1850  (uint32_t)slot_id,
1851  (uint32_t)session_id);
1852 
1853  t->mirroring_slots[slot_id] = session_id;
1854  t->mirroring_slots_mask |= 1LLU << slot_id;
1855 }
1856 
1857 static inline void
1858 __instr_recirculate_exec(struct rte_swx_pipeline *p __rte_unused,
1859  struct thread *t,
1860  const struct instruction *ip __rte_unused)
1861 {
1862  TRACE("[Thread %2u]: recirculate\n",
1863  p->thread_id);
1864 
1865  t->recirculate = 1;
1866 }
1867 
1868 static inline void
1869 __instr_recircid_exec(struct rte_swx_pipeline *p __rte_unused,
1870  struct thread *t,
1871  const struct instruction *ip)
1872 {
1873  TRACE("[Thread %2u]: recircid (pass %u)\n",
1874  p->thread_id,
1875  t->recirc_pass_id);
1876 
1877  /* Meta-data. */
1878  METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, t->recirc_pass_id);
1879 }
1880 
1881 /*
1882  * extract.
1883  */
1884 static inline void
1885 __instr_hdr_extract_many_exec(struct rte_swx_pipeline *p __rte_unused,
1886  struct thread *t,
1887  const struct instruction *ip,
1888  uint32_t n_extract)
1889 {
1890  uint64_t valid_headers = t->valid_headers;
1891  uint8_t *ptr = t->ptr;
1892  uint32_t offset = t->pkt.offset;
1893  uint32_t length = t->pkt.length;
1894  uint32_t i;
1895 
1896  for (i = 0; i < n_extract; i++) {
1897  uint32_t header_id = ip->io.hdr.header_id[i];
1898  uint32_t struct_id = ip->io.hdr.struct_id[i];
1899  uint32_t n_bytes = ip->io.hdr.n_bytes[i];
1900 
1901  TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
1902  p->thread_id,
1903  header_id,
1904  n_bytes);
1905 
1906  /* Headers. */
1907  t->structs[struct_id] = ptr;
1908  valid_headers = MASK64_BIT_SET(valid_headers, header_id);
1909 
1910  /* Packet. */
1911  offset += n_bytes;
1912  length -= n_bytes;
1913  ptr += n_bytes;
1914  }
1915 
1916  /* Headers. */
1917  t->valid_headers = valid_headers;
1918 
1919  /* Packet. */
1920  t->pkt.offset = offset;
1921  t->pkt.length = length;
1922  t->ptr = ptr;
1923 }
1924 
1925 static inline void
1926 __instr_hdr_extract_exec(struct rte_swx_pipeline *p,
1927  struct thread *t,
1928  const struct instruction *ip)
1929 {
1930  __instr_hdr_extract_many_exec(p, t, ip, 1);
1931 }
1932 
1933 static inline void
1934 __instr_hdr_extract2_exec(struct rte_swx_pipeline *p,
1935  struct thread *t,
1936  const struct instruction *ip)
1937 {
1938  TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
1939 
1940  __instr_hdr_extract_many_exec(p, t, ip, 2);
1941 }
1942 
1943 static inline void
1944 __instr_hdr_extract3_exec(struct rte_swx_pipeline *p,
1945  struct thread *t,
1946  const struct instruction *ip)
1947 {
1948  TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
1949 
1950  __instr_hdr_extract_many_exec(p, t, ip, 3);
1951 }
1952 
1953 static inline void
1954 __instr_hdr_extract4_exec(struct rte_swx_pipeline *p,
1955  struct thread *t,
1956  const struct instruction *ip)
1957 {
1958  TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
1959 
1960  __instr_hdr_extract_many_exec(p, t, ip, 4);
1961 }
1962 
1963 static inline void
1964 __instr_hdr_extract5_exec(struct rte_swx_pipeline *p,
1965  struct thread *t,
1966  const struct instruction *ip)
1967 {
1968  TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
1969 
1970  __instr_hdr_extract_many_exec(p, t, ip, 5);
1971 }
1972 
1973 static inline void
1974 __instr_hdr_extract6_exec(struct rte_swx_pipeline *p,
1975  struct thread *t,
1976  const struct instruction *ip)
1977 {
1978  TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
1979 
1980  __instr_hdr_extract_many_exec(p, t, ip, 6);
1981 }
1982 
1983 static inline void
1984 __instr_hdr_extract7_exec(struct rte_swx_pipeline *p,
1985  struct thread *t,
1986  const struct instruction *ip)
1987 {
1988  TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
1989 
1990  __instr_hdr_extract_many_exec(p, t, ip, 7);
1991 }
1992 
1993 static inline void
1994 __instr_hdr_extract8_exec(struct rte_swx_pipeline *p,
1995  struct thread *t,
1996  const struct instruction *ip)
1997 {
1998  TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
1999 
2000  __instr_hdr_extract_many_exec(p, t, ip, 8);
2001 }
2002 
2003 static inline void
2004 __instr_hdr_extract_m_exec(struct rte_swx_pipeline *p __rte_unused,
2005  struct thread *t,
2006  const struct instruction *ip)
2007 {
2008  uint64_t valid_headers = t->valid_headers;
2009  uint8_t *ptr = t->ptr;
2010  uint32_t offset = t->pkt.offset;
2011  uint32_t length = t->pkt.length;
2012 
2013  uint32_t n_bytes_last = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
2014  uint32_t header_id = ip->io.hdr.header_id[0];
2015  uint32_t struct_id = ip->io.hdr.struct_id[0];
2016  uint32_t n_bytes = ip->io.hdr.n_bytes[0];
2017 
2018  struct header_runtime *h = &t->headers[header_id];
2019 
2020  TRACE("[Thread %2u]: extract header %u (%u + %u bytes)\n",
2021  p->thread_id,
2022  header_id,
2023  n_bytes,
2024  n_bytes_last);
2025 
2026  n_bytes += n_bytes_last;
2027 
2028  /* Headers. */
2029  t->structs[struct_id] = ptr;
2030  t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2031  h->n_bytes = n_bytes;
2032 
2033  /* Packet. */
2034  t->pkt.offset = offset + n_bytes;
2035  t->pkt.length = length - n_bytes;
2036  t->ptr = ptr + n_bytes;
2037 }
2038 
2039 static inline void
2040 __instr_hdr_lookahead_exec(struct rte_swx_pipeline *p __rte_unused,
2041  struct thread *t,
2042  const struct instruction *ip)
2043 {
2044  uint64_t valid_headers = t->valid_headers;
2045  uint8_t *ptr = t->ptr;
2046 
2047  uint32_t header_id = ip->io.hdr.header_id[0];
2048  uint32_t struct_id = ip->io.hdr.struct_id[0];
2049 
2050  TRACE("[Thread %2u]: lookahead header %u\n",
2051  p->thread_id,
2052  header_id);
2053 
2054  /* Headers. */
2055  t->structs[struct_id] = ptr;
2056  t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2057 }
2058 
2059 /*
2060  * emit.
2061  */
2062 static inline void
2063 __instr_hdr_emit_many_exec(struct rte_swx_pipeline *p __rte_unused,
2064  struct thread *t,
2065  const struct instruction *ip,
2066  uint32_t n_emit)
2067 {
2068  uint64_t valid_headers = t->valid_headers;
2069  uint32_t n_headers_out = t->n_headers_out;
2070  struct header_out_runtime *ho = NULL;
2071  uint8_t *ho_ptr = NULL;
2072  uint32_t ho_nbytes = 0, i;
2073 
2074  for (i = 0; i < n_emit; i++) {
2075  uint32_t header_id = ip->io.hdr.header_id[i];
2076  uint32_t struct_id = ip->io.hdr.struct_id[i];
2077 
2078  struct header_runtime *hi = &t->headers[header_id];
2079  uint8_t *hi_ptr0 = hi->ptr0;
2080  uint32_t n_bytes = hi->n_bytes;
2081 
2082  uint8_t *hi_ptr = t->structs[struct_id];
2083 
2084  if (!MASK64_BIT_GET(valid_headers, header_id)) {
2085  TRACE("[Thread %2u]: emit header %u (invalid)\n",
2086  p->thread_id,
2087  header_id);
2088 
2089  continue;
2090  }
2091 
2092  TRACE("[Thread %2u]: emit header %u (valid)\n",
2093  p->thread_id,
2094  header_id);
2095 
2096  /* Headers. */
2097  if (!ho) {
2098  if (!n_headers_out) {
2099  ho = &t->headers_out[0];
2100 
2101  ho->ptr0 = hi_ptr0;
2102  ho->ptr = hi_ptr;
2103 
2104  ho_ptr = hi_ptr;
2105  ho_nbytes = n_bytes;
2106 
2107  n_headers_out = 1;
2108 
2109  continue;
2110  } else {
2111  ho = &t->headers_out[n_headers_out - 1];
2112 
2113  ho_ptr = ho->ptr;
2114  ho_nbytes = ho->n_bytes;
2115  }
2116  }
2117 
2118  if (ho_ptr + ho_nbytes == hi_ptr) {
2119  ho_nbytes += n_bytes;
2120  } else {
2121  ho->n_bytes = ho_nbytes;
2122 
2123  ho++;
2124  ho->ptr0 = hi_ptr0;
2125  ho->ptr = hi_ptr;
2126 
2127  ho_ptr = hi_ptr;
2128  ho_nbytes = n_bytes;
2129 
2130  n_headers_out++;
2131  }
2132  }
2133 
2134  if (ho)
2135  ho->n_bytes = ho_nbytes;
2136  t->n_headers_out = n_headers_out;
2137 }
2138 
2139 static inline void
2140 __instr_hdr_emit_exec(struct rte_swx_pipeline *p,
2141  struct thread *t,
2142  const struct instruction *ip)
2143 {
2144  __instr_hdr_emit_many_exec(p, t, ip, 1);
2145 }
2146 
2147 static inline void
2148 __instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p,
2149  struct thread *t,
2150  const struct instruction *ip)
2151 {
2152  TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
2153 
2154  __instr_hdr_emit_many_exec(p, t, ip, 1);
2155  __instr_tx_exec(p, t, ip);
2156 }
2157 
2158 static inline void
2159 __instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p,
2160  struct thread *t,
2161  const struct instruction *ip)
2162 {
2163  TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
2164 
2165  __instr_hdr_emit_many_exec(p, t, ip, 2);
2166  __instr_tx_exec(p, t, ip);
2167 }
2168 
2169 static inline void
2170 __instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p,
2171  struct thread *t,
2172  const struct instruction *ip)
2173 {
2174  TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
2175 
2176  __instr_hdr_emit_many_exec(p, t, ip, 3);
2177  __instr_tx_exec(p, t, ip);
2178 }
2179 
2180 static inline void
2181 __instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p,
2182  struct thread *t,
2183  const struct instruction *ip)
2184 {
2185  TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
2186 
2187  __instr_hdr_emit_many_exec(p, t, ip, 4);
2188  __instr_tx_exec(p, t, ip);
2189 }
2190 
2191 static inline void
2192 __instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p,
2193  struct thread *t,
2194  const struct instruction *ip)
2195 {
2196  TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
2197 
2198  __instr_hdr_emit_many_exec(p, t, ip, 5);
2199  __instr_tx_exec(p, t, ip);
2200 }
2201 
2202 static inline void
2203 __instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p,
2204  struct thread *t,
2205  const struct instruction *ip)
2206 {
2207  TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
2208 
2209  __instr_hdr_emit_many_exec(p, t, ip, 6);
2210  __instr_tx_exec(p, t, ip);
2211 }
2212 
2213 static inline void
2214 __instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p,
2215  struct thread *t,
2216  const struct instruction *ip)
2217 {
2218  TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
2219 
2220  __instr_hdr_emit_many_exec(p, t, ip, 7);
2221  __instr_tx_exec(p, t, ip);
2222 }
2223 
2224 static inline void
2225 __instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p,
2226  struct thread *t,
2227  const struct instruction *ip)
2228 {
2229  TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n", p->thread_id);
2230 
2231  __instr_hdr_emit_many_exec(p, t, ip, 8);
2232  __instr_tx_exec(p, t, ip);
2233 }
2234 
2235 /*
2236  * validate.
2237  */
2238 static inline void
2239 __instr_hdr_validate_exec(struct rte_swx_pipeline *p __rte_unused,
2240  struct thread *t,
2241  const struct instruction *ip)
2242 {
2243  uint32_t header_id = ip->valid.header_id;
2244  uint32_t struct_id = ip->valid.struct_id;
2245  uint64_t valid_headers = t->valid_headers;
2246  struct header_runtime *h = &t->headers[header_id];
2247 
2248  TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
2249 
2250  /* If this header is already valid, then its associated t->structs[] element is also valid
2251  * and therefore it should not be modified. It could point to the packet buffer (in case of
2252  * extracted header) and setting it to the default location (h->ptr0) would be incorrect.
2253  */
2254  if (MASK64_BIT_GET(valid_headers, header_id))
2255  return;
2256 
2257  /* Headers. */
2258  t->structs[struct_id] = h->ptr0;
2259  t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2260 }
2261 
2262 /*
2263  * invalidate.
2264  */
2265 static inline void
2266 __instr_hdr_invalidate_exec(struct rte_swx_pipeline *p __rte_unused,
2267  struct thread *t,
2268  const struct instruction *ip)
2269 {
2270  uint32_t header_id = ip->valid.header_id;
2271 
2272  TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
2273 
2274  /* Headers. */
2275  t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
2276 }
2277 
2278 /*
2279  * learn.
2280  */
2281 static inline void
2282 __instr_learn_exec(struct rte_swx_pipeline *p,
2283  struct thread *t,
2284  const struct instruction *ip)
2285 {
2286  uint64_t action_id = ip->learn.action_id;
2287  uint32_t mf_first_arg_offset = ip->learn.mf_first_arg_offset;
2288  uint32_t timeout_id = METADATA_READ(t, ip->learn.mf_timeout_id_offset,
2289  ip->learn.mf_timeout_id_n_bits);
2290  uint32_t learner_id = t->learner_id;
2291  struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2292  p->n_selectors + learner_id];
2293  struct learner_runtime *l = &t->learners[learner_id];
2294  struct learner_statistics *stats = &p->learner_stats[learner_id];
2295  uint32_t status;
2296 
2297  /* Table. */
2298  status = rte_swx_table_learner_add(ts->obj,
2299  l->mailbox,
2300  t->time,
2301  action_id,
2302  &t->metadata[mf_first_arg_offset],
2303  timeout_id);
2304 
2305  TRACE("[Thread %2u] learner %u learn %s\n",
2306  p->thread_id,
2307  learner_id,
2308  status ? "ok" : "error");
2309 
2310  stats->n_pkts_learn[status] += 1;
2311 }
2312 
2313 /*
2314  * rearm.
2315  */
2316 static inline void
2317 __instr_rearm_exec(struct rte_swx_pipeline *p,
2318  struct thread *t,
2319  const struct instruction *ip __rte_unused)
2320 {
2321  uint32_t learner_id = t->learner_id;
2322  struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2323  p->n_selectors + learner_id];
2324  struct learner_runtime *l = &t->learners[learner_id];
2325  struct learner_statistics *stats = &p->learner_stats[learner_id];
2326 
2327  /* Table. */
2328  rte_swx_table_learner_rearm(ts->obj, l->mailbox, t->time);
2329 
2330  TRACE("[Thread %2u] learner %u rearm\n",
2331  p->thread_id,
2332  learner_id);
2333 
2334  stats->n_pkts_rearm += 1;
2335 }
2336 
2337 static inline void
2338 __instr_rearm_new_exec(struct rte_swx_pipeline *p,
2339  struct thread *t,
2340  const struct instruction *ip)
2341 {
2342  uint32_t timeout_id = METADATA_READ(t, ip->learn.mf_timeout_id_offset,
2343  ip->learn.mf_timeout_id_n_bits);
2344  uint32_t learner_id = t->learner_id;
2345  struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2346  p->n_selectors + learner_id];
2347  struct learner_runtime *l = &t->learners[learner_id];
2348  struct learner_statistics *stats = &p->learner_stats[learner_id];
2349 
2350  /* Table. */
2351  rte_swx_table_learner_rearm_new(ts->obj, l->mailbox, t->time, timeout_id);
2352 
2353  TRACE("[Thread %2u] learner %u rearm with timeout ID %u\n",
2354  p->thread_id,
2355  learner_id,
2356  timeout_id);
2357 
2358  stats->n_pkts_rearm += 1;
2359 }
2360 
2361 /*
2362  * forget.
2363  */
2364 static inline void
2365 __instr_forget_exec(struct rte_swx_pipeline *p,
2366  struct thread *t,
2367  const struct instruction *ip __rte_unused)
2368 {
2369  uint32_t learner_id = t->learner_id;
2370  struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2371  p->n_selectors + learner_id];
2372  struct learner_runtime *l = &t->learners[learner_id];
2373  struct learner_statistics *stats = &p->learner_stats[learner_id];
2374 
2375  /* Table. */
2376  rte_swx_table_learner_delete(ts->obj, l->mailbox);
2377 
2378  TRACE("[Thread %2u] learner %u forget\n",
2379  p->thread_id,
2380  learner_id);
2381 
2382  stats->n_pkts_forget += 1;
2383 }
2384 
2385 /*
2386  * entryid.
2387  */
2388 static inline void
2389 __instr_entryid_exec(struct rte_swx_pipeline *p __rte_unused,
2390  struct thread *t,
2391  const struct instruction *ip)
2392 {
2393  TRACE("[Thread %2u]: entryid\n",
2394  p->thread_id);
2395 
2396  /* Meta-data. */
2397  METADATA_WRITE(t, ip->mov.dst.offset, ip->mov.dst.n_bits, t->entry_id);
2398 }
2399 
2400 /*
2401  * extern.
2402  */
2403 static inline uint32_t
2404 __instr_extern_obj_exec(struct rte_swx_pipeline *p __rte_unused,
2405  struct thread *t,
2406  const struct instruction *ip)
2407 {
2408  uint32_t obj_id = ip->ext_obj.ext_obj_id;
2409  uint32_t func_id = ip->ext_obj.func_id;
2410  struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
2411  rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
2412  uint32_t done;
2413 
2414  TRACE("[Thread %2u] extern obj %u member func %u\n",
2415  p->thread_id,
2416  obj_id,
2417  func_id);
2418 
2419  done = func(obj->obj, obj->mailbox);
2420 
2421  return done;
2422 }
2423 
2424 static inline uint32_t
2425 __instr_extern_func_exec(struct rte_swx_pipeline *p __rte_unused,
2426  struct thread *t,
2427  const struct instruction *ip)
2428 {
2429  uint32_t ext_func_id = ip->ext_func.ext_func_id;
2430  struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
2431  rte_swx_extern_func_t func = ext_func->func;
2432  uint32_t done;
2433 
2434  TRACE("[Thread %2u] extern func %u\n",
2435  p->thread_id,
2436  ext_func_id);
2437 
2438  done = func(ext_func->mailbox);
2439 
2440  return done;
2441 }
2442 
2443 /*
2444  * hash.
2445  */
2446 static inline void
2447 __instr_hash_func_exec(struct rte_swx_pipeline *p,
2448  struct thread *t,
2449  const struct instruction *ip)
2450 {
2451  uint32_t hash_func_id = ip->hash_func.hash_func_id;
2452  uint32_t dst_offset = ip->hash_func.dst.offset;
2453  uint32_t n_dst_bits = ip->hash_func.dst.n_bits;
2454  uint32_t src_struct_id = ip->hash_func.src.struct_id;
2455  uint32_t src_offset = ip->hash_func.src.offset;
2456  uint32_t n_src_bytes = ip->hash_func.src.n_bytes;
2457 
2458  struct hash_func_runtime *func = &p->hash_func_runtime[hash_func_id];
2459  uint8_t *src_ptr = t->structs[src_struct_id];
2460  uint32_t result;
2461 
2462  TRACE("[Thread %2u] hash %u\n",
2463  p->thread_id,
2464  hash_func_id);
2465 
2466  result = func->func(&src_ptr[src_offset], n_src_bytes, 0);
2467  METADATA_WRITE(t, dst_offset, n_dst_bits, result);
2468 }
2469 
2470 /*
2471  * mov.
2472  */
2473 static inline void
2474 __instr_mov_exec(struct rte_swx_pipeline *p __rte_unused,
2475  struct thread *t,
2476  const struct instruction *ip)
2477 {
2478  TRACE("[Thread %2u] mov\n", p->thread_id);
2479 
2480  MOV(t, ip);
2481 }
2482 
2483 static inline void
2484 __instr_mov_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2485  struct thread *t,
2486  const struct instruction *ip)
2487 {
2488  TRACE("[Thread %2u] mov (mh)\n", p->thread_id);
2489 
2490  MOV_MH(t, ip);
2491 }
2492 
2493 static inline void
2494 __instr_mov_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2495  struct thread *t,
2496  const struct instruction *ip)
2497 {
2498  TRACE("[Thread %2u] mov (hm)\n", p->thread_id);
2499 
2500  MOV_HM(t, ip);
2501 }
2502 
2503 static inline void
2504 __instr_mov_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2505  struct thread *t,
2506  const struct instruction *ip)
2507 {
2508  TRACE("[Thread %2u] mov (hh)\n", p->thread_id);
2509 
2510  MOV_HH(t, ip);
2511 }
2512 
2513 static inline void
2514 __instr_mov_dma_exec(struct rte_swx_pipeline *p __rte_unused,
2515  struct thread *t,
2516  const struct instruction *ip)
2517 {
2518  uint8_t *dst_struct = t->structs[ip->mov.dst.struct_id];
2519  uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->mov.dst.offset];
2520  uint32_t *dst32_ptr;
2521  uint16_t *dst16_ptr;
2522  uint8_t *dst8_ptr;
2523 
2524  uint8_t *src_struct = t->structs[ip->mov.src.struct_id];
2525  uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->mov.src.offset];
2526  uint32_t *src32_ptr;
2527  uint16_t *src16_ptr;
2528  uint8_t *src8_ptr;
2529 
2530  uint32_t n = ip->mov.dst.n_bits >> 3, i;
2531 
2532  TRACE("[Thread %2u] mov (dma) %u bytes\n", p->thread_id, n);
2533 
2534  /* 8-byte transfers. */
2535  for (i = 0; i < n >> 3; i++)
2536  *dst64_ptr++ = *src64_ptr++;
2537 
2538  /* 4-byte transfers. */
2539  n &= 7;
2540  dst32_ptr = (uint32_t *)dst64_ptr;
2541  src32_ptr = (uint32_t *)src64_ptr;
2542 
2543  for (i = 0; i < n >> 2; i++)
2544  *dst32_ptr++ = *src32_ptr++;
2545 
2546  /* 2-byte transfers. */
2547  n &= 3;
2548  dst16_ptr = (uint16_t *)dst32_ptr;
2549  src16_ptr = (uint16_t *)src32_ptr;
2550 
2551  for (i = 0; i < n >> 1; i++)
2552  *dst16_ptr++ = *src16_ptr++;
2553 
2554  /* 1-byte transfer. */
2555  n &= 1;
2556  dst8_ptr = (uint8_t *)dst16_ptr;
2557  src8_ptr = (uint8_t *)src16_ptr;
2558  if (n)
2559  *dst8_ptr = *src8_ptr;
2560 }
2561 
2562 static inline void
2563 __instr_mov_128_exec(struct rte_swx_pipeline *p __rte_unused,
2564  struct thread *t,
2565  const struct instruction *ip)
2566 {
2567  uint8_t *dst_struct = t->structs[ip->mov.dst.struct_id];
2568  uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->mov.dst.offset];
2569 
2570  uint8_t *src_struct = t->structs[ip->mov.src.struct_id];
2571  uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->mov.src.offset];
2572 
2573  TRACE("[Thread %2u] mov (128)\n", p->thread_id);
2574 
2575  dst64_ptr[0] = src64_ptr[0];
2576  dst64_ptr[1] = src64_ptr[1];
2577 }
2578 
2579 static inline void
2580 __instr_mov_i_exec(struct rte_swx_pipeline *p __rte_unused,
2581  struct thread *t,
2582  const struct instruction *ip)
2583 {
2584  TRACE("[Thread %2u] mov m.f %" PRIx64 "\n", p->thread_id, ip->mov.src_val);
2585 
2586  MOV_I(t, ip);
2587 }
2588 
2589 /*
2590  * dma.
2591  */
2592 static inline void
2593 __instr_dma_ht_many_exec(struct rte_swx_pipeline *p __rte_unused,
2594  struct thread *t,
2595  const struct instruction *ip,
2596  uint32_t n_dma)
2597 {
2598  uint8_t *action_data = t->structs[0];
2599  uint64_t valid_headers = t->valid_headers;
2600  uint32_t i;
2601 
2602  for (i = 0; i < n_dma; i++) {
2603  uint32_t header_id = ip->dma.dst.header_id[i];
2604  uint32_t struct_id = ip->dma.dst.struct_id[i];
2605  uint32_t offset = ip->dma.src.offset[i];
2606  uint32_t n_bytes = ip->dma.n_bytes[i];
2607 
2608  struct header_runtime *h = &t->headers[header_id];
2609  uint8_t *h_ptr0 = h->ptr0;
2610  uint8_t *h_ptr = t->structs[struct_id];
2611 
2612  void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
2613  h_ptr : h_ptr0;
2614  void *src = &action_data[offset];
2615 
2616  TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
2617 
2618  /* Headers. */
2619  memcpy(dst, src, n_bytes);
2620  t->structs[struct_id] = dst;
2621  valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2622  }
2623 
2624  t->valid_headers = valid_headers;
2625 }
2626 
2627 static inline void
2628 __instr_dma_ht_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2629 {
2630  __instr_dma_ht_many_exec(p, t, ip, 1);
2631 }
2632 
2633 static inline void
2634 __instr_dma_ht2_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2635 {
2636  TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
2637 
2638  __instr_dma_ht_many_exec(p, t, ip, 2);
2639 }
2640 
2641 static inline void
2642 __instr_dma_ht3_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2643 {
2644  TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
2645 
2646  __instr_dma_ht_many_exec(p, t, ip, 3);
2647 }
2648 
2649 static inline void
2650 __instr_dma_ht4_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2651 {
2652  TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
2653 
2654  __instr_dma_ht_many_exec(p, t, ip, 4);
2655 }
2656 
2657 static inline void
2658 __instr_dma_ht5_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2659 {
2660  TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
2661 
2662  __instr_dma_ht_many_exec(p, t, ip, 5);
2663 }
2664 
2665 static inline void
2666 __instr_dma_ht6_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2667 {
2668  TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
2669 
2670  __instr_dma_ht_many_exec(p, t, ip, 6);
2671 }
2672 
2673 static inline void
2674 __instr_dma_ht7_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2675 {
2676  TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
2677 
2678  __instr_dma_ht_many_exec(p, t, ip, 7);
2679 }
2680 
2681 static inline void
2682 __instr_dma_ht8_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2683 {
2684  TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
2685 
2686  __instr_dma_ht_many_exec(p, t, ip, 8);
2687 }
2688 
2689 /*
2690  * alu.
2691  */
2692 static inline void
2693 __instr_alu_add_exec(struct rte_swx_pipeline *p __rte_unused,
2694  struct thread *t,
2695  const struct instruction *ip)
2696 {
2697  TRACE("[Thread %2u] add\n", p->thread_id);
2698 
2699  ALU(t, ip, +);
2700 }
2701 
2702 static inline void
2703 __instr_alu_add_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2704  struct thread *t,
2705  const struct instruction *ip)
2706 {
2707  TRACE("[Thread %2u] add (mh)\n", p->thread_id);
2708 
2709  ALU_MH(t, ip, +);
2710 }
2711 
2712 static inline void
2713 __instr_alu_add_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2714  struct thread *t,
2715  const struct instruction *ip)
2716 {
2717  TRACE("[Thread %2u] add (hm)\n", p->thread_id);
2718 
2719  ALU_HM(t, ip, +);
2720 }
2721 
2722 static inline void
2723 __instr_alu_add_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2724  struct thread *t,
2725  const struct instruction *ip)
2726 {
2727  TRACE("[Thread %2u] add (hh)\n", p->thread_id);
2728 
2729  ALU_HH(t, ip, +);
2730 }
2731 
2732 static inline void
2733 __instr_alu_add_mi_exec(struct rte_swx_pipeline *p __rte_unused,
2734  struct thread *t,
2735  const struct instruction *ip)
2736 {
2737  TRACE("[Thread %2u] add (mi)\n", p->thread_id);
2738 
2739  ALU_MI(t, ip, +);
2740 }
2741 
2742 static inline void
2743 __instr_alu_add_hi_exec(struct rte_swx_pipeline *p __rte_unused,
2744  struct thread *t,
2745  const struct instruction *ip)
2746 {
2747  TRACE("[Thread %2u] add (hi)\n", p->thread_id);
2748 
2749  ALU_HI(t, ip, +);
2750 }
2751 
2752 static inline void
2753 __instr_alu_sub_exec(struct rte_swx_pipeline *p __rte_unused,
2754  struct thread *t,
2755  const struct instruction *ip)
2756 {
2757  TRACE("[Thread %2u] sub\n", p->thread_id);
2758 
2759  ALU(t, ip, -);
2760 }
2761 
2762 static inline void
2763 __instr_alu_sub_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2764  struct thread *t,
2765  const struct instruction *ip)
2766 {
2767  TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
2768 
2769  ALU_MH(t, ip, -);
2770 }
2771 
2772 static inline void
2773 __instr_alu_sub_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2774  struct thread *t,
2775  const struct instruction *ip)
2776 {
2777  TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
2778 
2779  ALU_HM(t, ip, -);
2780 }
2781 
2782 static inline void
2783 __instr_alu_sub_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2784  struct thread *t,
2785  const struct instruction *ip)
2786 {
2787  TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
2788 
2789  ALU_HH(t, ip, -);
2790 }
2791 
2792 static inline void
2793 __instr_alu_sub_mi_exec(struct rte_swx_pipeline *p __rte_unused,
2794  struct thread *t,
2795  const struct instruction *ip)
2796 {
2797  TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
2798 
2799  ALU_MI(t, ip, -);
2800 }
2801 
2802 static inline void
2803 __instr_alu_sub_hi_exec(struct rte_swx_pipeline *p __rte_unused,
2804  struct thread *t,
2805  const struct instruction *ip)
2806 {
2807  TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
2808 
2809  ALU_HI(t, ip, -);
2810 }
2811 
2812 static inline void
2813 __instr_alu_shl_exec(struct rte_swx_pipeline *p __rte_unused,
2814  struct thread *t,
2815  const struct instruction *ip)
2816 {
2817  TRACE("[Thread %2u] shl\n", p->thread_id);
2818 
2819  ALU(t, ip, <<);
2820 }
2821 
2822 static inline void
2823 __instr_alu_shl_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2824  struct thread *t,
2825  const struct instruction *ip)
2826 {
2827  TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
2828 
2829  ALU_MH(t, ip, <<);
2830 }
2831 
2832 static inline void
2833 __instr_alu_shl_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2834  struct thread *t,
2835  const struct instruction *ip)
2836 {
2837  TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
2838 
2839  ALU_HM(t, ip, <<);
2840 }
2841 
2842 static inline void
2843 __instr_alu_shl_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2844  struct thread *t,
2845  const struct instruction *ip)
2846 {
2847  TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
2848 
2849  ALU_HH(t, ip, <<);
2850 }
2851 
2852 static inline void
2853 __instr_alu_shl_mi_exec(struct rte_swx_pipeline *p __rte_unused,
2854  struct thread *t,
2855  const struct instruction *ip)
2856 {
2857  TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
2858 
2859  ALU_MI(t, ip, <<);
2860 }
2861 
2862 static inline void
2863 __instr_alu_shl_hi_exec(struct rte_swx_pipeline *p __rte_unused,
2864  struct thread *t,
2865  const struct instruction *ip)
2866 {
2867  TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
2868 
2869  ALU_HI(t, ip, <<);
2870 }
2871 
2872 static inline void
2873 __instr_alu_shr_exec(struct rte_swx_pipeline *p __rte_unused,
2874  struct thread *t,
2875  const struct instruction *ip)
2876 {
2877  TRACE("[Thread %2u] shr\n", p->thread_id);
2878 
2879  ALU(t, ip, >>);
2880 }
2881 
2882 static inline void
2883 __instr_alu_shr_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2884  struct thread *t,
2885  const struct instruction *ip)
2886 {
2887  TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
2888 
2889  ALU_MH(t, ip, >>);
2890 }
2891 
2892 static inline void
2893 __instr_alu_shr_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2894  struct thread *t,
2895  const struct instruction *ip)
2896 {
2897  TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
2898 
2899  ALU_HM(t, ip, >>);
2900 }
2901 
2902 static inline void
2903 __instr_alu_shr_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2904  struct thread *t,
2905  const struct instruction *ip)
2906 {
2907  TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
2908 
2909  ALU_HH(t, ip, >>);
2910 }
2911 
2912 static inline void
2913 __instr_alu_shr_mi_exec(struct rte_swx_pipeline *p __rte_unused,
2914  struct thread *t,
2915  const struct instruction *ip)
2916 {
2917  TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
2918 
2919  /* Structs. */
2920  ALU_MI(t, ip, >>);
2921 }
2922 
2923 static inline void
2924 __instr_alu_shr_hi_exec(struct rte_swx_pipeline *p __rte_unused,
2925  struct thread *t,
2926  const struct instruction *ip)
2927 {
2928  TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
2929 
2930  ALU_HI(t, ip, >>);
2931 }
2932 
2933 static inline void
2934 __instr_alu_and_exec(struct rte_swx_pipeline *p __rte_unused,
2935  struct thread *t,
2936  const struct instruction *ip)
2937 {
2938  TRACE("[Thread %2u] and\n", p->thread_id);
2939 
2940  ALU(t, ip, &);
2941 }
2942 
2943 static inline void
2944 __instr_alu_and_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2945  struct thread *t,
2946  const struct instruction *ip)
2947 {
2948  TRACE("[Thread %2u] and (mh)\n", p->thread_id);
2949 
2950  ALU_MH(t, ip, &);
2951 }
2952 
2953 static inline void
2954 __instr_alu_and_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2955  struct thread *t,
2956  const struct instruction *ip)
2957 {
2958  TRACE("[Thread %2u] and (hm)\n", p->thread_id);
2959 
2960  ALU_HM_FAST(t, ip, &);
2961 }
2962 
2963 static inline void
2964 __instr_alu_and_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2965  struct thread *t,
2966  const struct instruction *ip)
2967 {
2968  TRACE("[Thread %2u] and (hh)\n", p->thread_id);
2969 
2970  ALU_HH_FAST(t, ip, &);
2971 }
2972 
2973 static inline void
2974 __instr_alu_and_i_exec(struct rte_swx_pipeline *p __rte_unused,
2975  struct thread *t,
2976  const struct instruction *ip)
2977 {
2978  TRACE("[Thread %2u] and (i)\n", p->thread_id);
2979 
2980  ALU_I(t, ip, &);
2981 }
2982 
2983 static inline void
2984 __instr_alu_or_exec(struct rte_swx_pipeline *p __rte_unused,
2985  struct thread *t,
2986  const struct instruction *ip)
2987 {
2988  TRACE("[Thread %2u] or\n", p->thread_id);
2989 
2990  ALU(t, ip, |);
2991 }
2992 
2993 static inline void
2994 __instr_alu_or_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2995  struct thread *t,
2996  const struct instruction *ip)
2997 {
2998  TRACE("[Thread %2u] or (mh)\n", p->thread_id);
2999 
3000  ALU_MH(t, ip, |);
3001 }
3002 
3003 static inline void
3004 __instr_alu_or_hm_exec(struct rte_swx_pipeline *p __rte_unused,
3005  struct thread *t,
3006  const struct instruction *ip)
3007 {
3008  TRACE("[Thread %2u] or (hm)\n", p->thread_id);
3009 
3010  ALU_HM_FAST(t, ip, |);
3011 }
3012 
3013 static inline void
3014 __instr_alu_or_hh_exec(struct rte_swx_pipeline *p __rte_unused,
3015  struct thread *t,
3016  const struct instruction *ip)
3017 {
3018  TRACE("[Thread %2u] or (hh)\n", p->thread_id);
3019 
3020  ALU_HH_FAST(t, ip, |);
3021 }
3022 
3023 static inline void
3024 __instr_alu_or_i_exec(struct rte_swx_pipeline *p __rte_unused,
3025  struct thread *t,
3026  const struct instruction *ip)
3027 {
3028  TRACE("[Thread %2u] or (i)\n", p->thread_id);
3029 
3030  ALU_I(t, ip, |);
3031 }
3032 
3033 static inline void
3034 __instr_alu_xor_exec(struct rte_swx_pipeline *p __rte_unused,
3035  struct thread *t,
3036  const struct instruction *ip)
3037 {
3038  TRACE("[Thread %2u] xor\n", p->thread_id);
3039 
3040  ALU(t, ip, ^);
3041 }
3042 
3043 static inline void
3044 __instr_alu_xor_mh_exec(struct rte_swx_pipeline *p __rte_unused,
3045  struct thread *t,
3046  const struct instruction *ip)
3047 {
3048  TRACE("[Thread %2u] xor (mh)\n", p->thread_id);
3049 
3050  ALU_MH(t, ip, ^);
3051 }
3052 
3053 static inline void
3054 __instr_alu_xor_hm_exec(struct rte_swx_pipeline *p __rte_unused,
3055  struct thread *t,
3056  const struct instruction *ip)
3057 {
3058  TRACE("[Thread %2u] xor (hm)\n", p->thread_id);
3059 
3060  ALU_HM_FAST(t, ip, ^);
3061 }
3062 
3063 static inline void
3064 __instr_alu_xor_hh_exec(struct rte_swx_pipeline *p __rte_unused,
3065  struct thread *t,
3066  const struct instruction *ip)
3067 {
3068  TRACE("[Thread %2u] xor (hh)\n", p->thread_id);
3069 
3070  ALU_HH_FAST(t, ip, ^);
3071 }
3072 
3073 static inline void
3074 __instr_alu_xor_i_exec(struct rte_swx_pipeline *p __rte_unused,
3075  struct thread *t,
3076  const struct instruction *ip)
3077 {
3078  TRACE("[Thread %2u] xor (i)\n", p->thread_id);
3079 
3080  ALU_I(t, ip, ^);
3081 }
3082 
3083 static inline void
3084 __instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p __rte_unused,
3085  struct thread *t,
3086  const struct instruction *ip)
3087 {
3088  uint8_t *dst_struct, *src_struct;
3089  uint16_t *dst16_ptr, dst;
3090  uint64_t *src64_ptr, src64, src64_mask, src;
3091  uint64_t r;
3092 
3093  TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
3094 
3095  /* Structs. */
3096  dst_struct = t->structs[ip->alu.dst.struct_id];
3097  dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3098  dst = *dst16_ptr;
3099 
3100  src_struct = t->structs[ip->alu.src.struct_id];
3101  src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3102  src64 = *src64_ptr;
3103  src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3104  src = src64 & src64_mask;
3105 
3106  /* Initialize the result with destination 1's complement. */
3107  r = dst;
3108  r = ~r & 0xFFFF;
3109 
3110  /* The first input (r) is a 16-bit number. The second and the third
3111  * inputs are 32-bit numbers. In the worst case scenario, the sum of the
3112  * three numbers (output r) is a 34-bit number.
3113  */
3114  r += (src >> 32) + (src & 0xFFFFFFFF);
3115 
3116  /* The first input is a 16-bit number. The second input is an 18-bit
3117  * number. In the worst case scenario, the sum of the two numbers is a
3118  * 19-bit number.
3119  */
3120  r = (r & 0xFFFF) + (r >> 16);
3121 
3122  /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3123  * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
3124  */
3125  r = (r & 0xFFFF) + (r >> 16);
3126 
3127  /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3128  * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3129  * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
3130  * therefore the output r is always a 16-bit number.
3131  */
3132  r = (r & 0xFFFF) + (r >> 16);
3133 
3134  /* Apply 1's complement to the result. */
3135  r = ~r & 0xFFFF;
3136  r = r ? r : 0xFFFF;
3137 
3138  *dst16_ptr = (uint16_t)r;
3139 }
3140 
3141 static inline void
3142 __instr_alu_cksub_field_exec(struct rte_swx_pipeline *p __rte_unused,
3143  struct thread *t,
3144  const struct instruction *ip)
3145 {
3146  uint8_t *dst_struct, *src_struct;
3147  uint16_t *dst16_ptr, dst;
3148  uint64_t *src64_ptr, src64, src64_mask, src;
3149  uint64_t r;
3150 
3151  TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
3152 
3153  /* Structs. */
3154  dst_struct = t->structs[ip->alu.dst.struct_id];
3155  dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3156  dst = *dst16_ptr;
3157 
3158  src_struct = t->structs[ip->alu.src.struct_id];
3159  src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3160  src64 = *src64_ptr;
3161  src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3162  src = src64 & src64_mask;
3163 
3164  /* Initialize the result with destination 1's complement. */
3165  r = dst;
3166  r = ~r & 0xFFFF;
3167 
3168  /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
3169  * the following sequence of operations in 2's complement arithmetic:
3170  * a '- b = (a - b) % 0xFFFF.
3171  *
3172  * In order to prevent an underflow for the below subtraction, in which
3173  * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
3174  * minuend), we first add a multiple of the 0xFFFF modulus to the
3175  * minuend. The number we add to the minuend needs to be a 34-bit number
3176  * or higher, so for readability reasons we picked the 36-bit multiple.
3177  * We are effectively turning the 16-bit minuend into a 36-bit number:
3178  * (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
3179  */
3180  r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
3181 
3182  /* A 33-bit number is subtracted from a 36-bit number (the input r). The
3183  * result (the output r) is a 36-bit number.
3184  */
3185  r -= (src >> 32) + (src & 0xFFFFFFFF);
3186 
3187  /* The first input is a 16-bit number. The second input is a 20-bit
3188  * number. Their sum is a 21-bit number.
3189  */
3190  r = (r & 0xFFFF) + (r >> 16);
3191 
3192  /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3193  * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
3194  */
3195  r = (r & 0xFFFF) + (r >> 16);
3196 
3197  /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3198  * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3199  * 0x1001E), the output r is (0 .. 31). So no carry bit can be
3200  * generated, therefore the output r is always a 16-bit number.
3201  */
3202  r = (r & 0xFFFF) + (r >> 16);
3203 
3204  /* Apply 1's complement to the result. */
3205  r = ~r & 0xFFFF;
3206  r = r ? r : 0xFFFF;
3207 
3208  *dst16_ptr = (uint16_t)r;
3209 }
3210 
3211 static inline void
3212 __instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p __rte_unused,
3213  struct thread *t,
3214  const struct instruction *ip)
3215 {
3216  uint8_t *dst_struct, *src_struct;
3217  uint16_t *dst16_ptr, dst;
3218  uint32_t *src32_ptr;
3219  uint64_t r0, r1;
3220 
3221  TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
3222 
3223  /* Structs. */
3224  dst_struct = t->structs[ip->alu.dst.struct_id];
3225  dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3226  dst = *dst16_ptr;
3227 
3228  src_struct = t->structs[ip->alu.src.struct_id];
3229  src32_ptr = (uint32_t *)&src_struct[0];
3230 
3231  /* Initialize the result with destination 1's complement. */
3232  r0 = dst;
3233  r0 = ~r0 & 0xFFFF;
3234 
3235  r0 += src32_ptr[0]; /* The output r0 is a 33-bit number. */
3236  r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
3237  r0 += src32_ptr[2]; /* The output r0 is a 34-bit number. */
3238  r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
3239  r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
3240 
3241  /* The first input is a 16-bit number. The second input is a 19-bit
3242  * number. Their sum is a 20-bit number.
3243  */
3244  r0 = (r0 & 0xFFFF) + (r0 >> 16);
3245 
3246  /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3247  * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
3248  */
3249  r0 = (r0 & 0xFFFF) + (r0 >> 16);
3250 
3251  /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3252  * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3253  * 0x1000E), the output r is (0 .. 15). So no carry bit can be
3254  * generated, therefore the output r is always a 16-bit number.
3255  */
3256  r0 = (r0 & 0xFFFF) + (r0 >> 16);
3257 
3258  /* Apply 1's complement to the result. */
3259  r0 = ~r0 & 0xFFFF;
3260  r0 = r0 ? r0 : 0xFFFF;
3261 
3262  *dst16_ptr = (uint16_t)r0;
3263 }
3264 
3265 static inline void
3266 __instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p __rte_unused,
3267  struct thread *t,
3268  const struct instruction *ip)
3269 {
3270  uint32_t src_header_id = ip->alu.src.n_bits; /* The src header ID is stored here. */
3271  uint32_t n_src_header_bytes = t->headers[src_header_id].n_bytes;
3272  uint8_t *dst_struct, *src_struct;
3273  uint16_t *dst16_ptr, dst;
3274  uint32_t *src32_ptr;
3275  uint64_t r;
3276  uint32_t i;
3277 
3278  if (n_src_header_bytes == 20) {
3279  __instr_alu_ckadd_struct20_exec(p, t, ip);
3280  return;
3281  }
3282 
3283  TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
3284 
3285  /* Structs. */
3286  dst_struct = t->structs[ip->alu.dst.struct_id];
3287  dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3288  dst = *dst16_ptr;
3289 
3290  src_struct = t->structs[ip->alu.src.struct_id];
3291  src32_ptr = (uint32_t *)&src_struct[0];
3292 
3293  /* Initialize the result with destination 1's complement. */
3294  r = dst;
3295  r = ~r & 0xFFFF;
3296 
3297  /* The max number of 32-bit words in a 32K-byte header is 2^13.
3298  * Therefore, in the worst case scenario, a 45-bit number is added to a
3299  * 16-bit number (the input r), so the output r is 46-bit number.
3300  */
3301  for (i = 0; i < n_src_header_bytes / 4; i++, src32_ptr++)
3302  r += *src32_ptr;
3303 
3304  /* The first input is a 16-bit number. The second input is a 30-bit
3305  * number. Their sum is a 31-bit number.
3306  */
3307  r = (r & 0xFFFF) + (r >> 16);
3308 
3309  /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3310  * a 15-bit number (0 .. 0x7FFF). The sum is a 17-bit number (0 .. 0x17FFE).
3311  */
3312  r = (r & 0xFFFF) + (r >> 16);
3313 
3314  /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3315  * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3316  * 0x17FFE), the output r is (0 .. 0x7FFF). So no carry bit can be
3317  * generated, therefore the output r is always a 16-bit number.
3318  */
3319  r = (r & 0xFFFF) + (r >> 16);
3320 
3321  /* Apply 1's complement to the result. */
3322  r = ~r & 0xFFFF;
3323  r = r ? r : 0xFFFF;
3324 
3325  *dst16_ptr = (uint16_t)r;
3326 }
3327 
3328 /*
3329  * Register array.
3330  */
3331 static inline uint64_t *
3332 instr_regarray_regarray(struct rte_swx_pipeline *p, const struct instruction *ip)
3333 {
3334  struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
3335  return r->regarray;
3336 }
3337 
3338 static inline uint64_t
3339 instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3340 {
3341  struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
3342 
3343  uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
3344  uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
3345  uint64_t idx64 = *idx64_ptr;
3346  uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
3347  uint64_t idx = idx64 & idx64_mask & r->size_mask;
3348 
3349  return idx;
3350 }
3351 
3352 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3353 
3354 static inline uint64_t
3355 instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3356 {
3357  struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
3358 
3359  uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
3360  uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
3361  uint64_t idx64 = *idx64_ptr;
3362  uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
3363 
3364  return idx;
3365 }
3366 
3367 #else
3368 
3369 #define instr_regarray_idx_nbo instr_regarray_idx_hbo
3370 
3371 #endif
3372 
3373 static inline uint64_t
3374 instr_regarray_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip)
3375 {
3376  struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
3377 
3378  uint64_t idx = ip->regarray.idx_val & r->size_mask;
3379 
3380  return idx;
3381 }
3382 
3383 static inline uint64_t
3384 instr_regarray_src_hbo(struct thread *t, const struct instruction *ip)
3385 {
3386  uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
3387  uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
3388  uint64_t src64 = *src64_ptr;
3389  uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
3390  uint64_t src = src64 & src64_mask;
3391 
3392  return src;
3393 }
3394 
3395 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3396 
3397 static inline uint64_t
3398 instr_regarray_src_nbo(struct thread *t, const struct instruction *ip)
3399 {
3400  uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
3401  uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
3402  uint64_t src64 = *src64_ptr;
3403  uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
3404 
3405  return src;
3406 }
3407 
3408 #else
3409 
3410 #define instr_regarray_src_nbo instr_regarray_src_hbo
3411 
3412 #endif
3413 
3414 static inline void
3415 instr_regarray_dst_hbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src)
3416 {
3417  uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
3418  uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
3419  uint64_t dst64 = *dst64_ptr;
3420  uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
3421 
3422  *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
3423 
3424 }
3425 
3426 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3427 
3428 static inline void
3429 instr_regarray_dst_nbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src)
3430 {
3431  uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
3432  uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
3433  uint64_t dst64 = *dst64_ptr;
3434  uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
3435 
3436  src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
3437  *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
3438 }
3439 
3440 #else
3441 
3442 #define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
3443 
3444 #endif
3445 
3446 static inline void
3447 __instr_regprefetch_rh_exec(struct rte_swx_pipeline *p,
3448  struct thread *t,
3449  const struct instruction *ip)
3450 {
3451  uint64_t *regarray, idx;
3452 
3453  TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
3454 
3455  regarray = instr_regarray_regarray(p, ip);
3456  idx = instr_regarray_idx_nbo(p, t, ip);
3457  rte_prefetch0(&regarray[idx]);
3458 }
3459 
3460 static inline void
3461 __instr_regprefetch_rm_exec(struct rte_swx_pipeline *p,
3462  struct thread *t,
3463  const struct instruction *ip)
3464 {
3465  uint64_t *regarray, idx;
3466 
3467  TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
3468 
3469  regarray = instr_regarray_regarray(p, ip);
3470  idx = instr_regarray_idx_hbo(p, t, ip);
3471  rte_prefetch0(&regarray[idx]);
3472 }
3473 
3474 static inline void
3475 __instr_regprefetch_ri_exec(struct rte_swx_pipeline *p,
3476  struct thread *t __rte_unused,
3477  const struct instruction *ip)
3478 {
3479  uint64_t *regarray, idx;
3480 
3481  TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
3482 
3483  regarray = instr_regarray_regarray(p, ip);
3484  idx = instr_regarray_idx_imm(p, ip);
3485  rte_prefetch0(&regarray[idx]);
3486 }
3487 
3488 static inline void
3489 __instr_regrd_hrh_exec(struct rte_swx_pipeline *p,
3490  struct thread *t,
3491  const struct instruction *ip)
3492 {
3493  uint64_t *regarray, idx;
3494 
3495  TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
3496 
3497  regarray = instr_regarray_regarray(p, ip);
3498  idx = instr_regarray_idx_nbo(p, t, ip);
3499  instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
3500 }
3501 
3502 static inline void
3503 __instr_regrd_hrm_exec(struct rte_swx_pipeline *p,
3504  struct thread *t,
3505  const struct instruction *ip)
3506 {
3507  uint64_t *regarray, idx;
3508 
3509  TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
3510 
3511  /* Structs. */
3512  regarray = instr_regarray_regarray(p, ip);
3513  idx = instr_regarray_idx_hbo(p, t, ip);
3514  instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
3515 }
3516 
3517 static inline void
3518 __instr_regrd_mrh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3519 {
3520  uint64_t *regarray, idx;
3521 
3522  TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
3523 
3524  regarray = instr_regarray_regarray(p, ip);
3525  idx = instr_regarray_idx_nbo(p, t, ip);
3526  instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
3527 }
3528 
3529 static inline void
3530 __instr_regrd_mrm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3531 {
3532  uint64_t *regarray, idx;
3533 
3534  TRACE("[Thread %2u] regrd (m = r[m])\n", p->thread_id);
3535 
3536  regarray = instr_regarray_regarray(p, ip);
3537  idx = instr_regarray_idx_hbo(p, t, ip);
3538  instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
3539 }
3540 
3541 static inline void
3542 __instr_regrd_hri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3543 {
3544  uint64_t *regarray, idx;
3545 
3546  TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
3547 
3548  regarray = instr_regarray_regarray(p, ip);
3549  idx = instr_regarray_idx_imm(p, ip);
3550  instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
3551 }
3552 
3553 static inline void
3554 __instr_regrd_mri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3555 {
3556  uint64_t *regarray, idx;
3557 
3558  TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
3559 
3560  regarray = instr_regarray_regarray(p, ip);
3561  idx = instr_regarray_idx_imm(p, ip);
3562  instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
3563 }
3564 
3565 static inline void
3566 __instr_regwr_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3567 {
3568  uint64_t *regarray, idx, src;
3569 
3570  TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
3571 
3572  regarray = instr_regarray_regarray(p, ip);
3573  idx = instr_regarray_idx_nbo(p, t, ip);
3574  src = instr_regarray_src_nbo(t, ip);
3575  regarray[idx] = src;
3576 }
3577 
3578 static inline void
3579 __instr_regwr_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3580 {
3581  uint64_t *regarray, idx, src;
3582 
3583  TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
3584 
3585  regarray = instr_regarray_regarray(p, ip);
3586  idx = instr_regarray_idx_nbo(p, t, ip);
3587  src = instr_regarray_src_hbo(t, ip);
3588  regarray[idx] = src;
3589 }
3590 
3591 static inline void
3592 __instr_regwr_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3593 {
3594  uint64_t *regarray, idx, src;
3595 
3596  TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
3597 
3598  regarray = instr_regarray_regarray(p, ip);
3599  idx = instr_regarray_idx_hbo(p, t, ip);
3600  src = instr_regarray_src_nbo(t, ip);
3601  regarray[idx] = src;
3602 }
3603 
3604 static inline void
3605 __instr_regwr_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3606 {
3607  uint64_t *regarray, idx, src;
3608 
3609  TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
3610 
3611  regarray = instr_regarray_regarray(p, ip);
3612  idx = instr_regarray_idx_hbo(p, t, ip);
3613  src = instr_regarray_src_hbo(t, ip);
3614  regarray[idx] = src;
3615 }
3616 
3617 static inline void
3618 __instr_regwr_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3619 {
3620  uint64_t *regarray, idx, src;
3621 
3622  TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
3623 
3624  regarray = instr_regarray_regarray(p, ip);
3625  idx = instr_regarray_idx_nbo(p, t, ip);
3626  src = ip->regarray.dstsrc_val;
3627  regarray[idx] = src;
3628 }
3629 
3630 static inline void
3631 __instr_regwr_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3632 {
3633  uint64_t *regarray, idx, src;
3634 
3635  TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
3636 
3637  regarray = instr_regarray_regarray(p, ip);
3638  idx = instr_regarray_idx_hbo(p, t, ip);
3639  src = ip->regarray.dstsrc_val;
3640  regarray[idx] = src;
3641 }
3642 
3643 static inline void
3644 __instr_regwr_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3645 {
3646  uint64_t *regarray, idx, src;
3647 
3648  TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
3649 
3650  regarray = instr_regarray_regarray(p, ip);
3651  idx = instr_regarray_idx_imm(p, ip);
3652  src = instr_regarray_src_nbo(t, ip);
3653  regarray[idx] = src;
3654 }
3655 
3656 static inline void
3657 __instr_regwr_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3658 {
3659  uint64_t *regarray, idx, src;
3660 
3661  TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
3662 
3663  regarray = instr_regarray_regarray(p, ip);
3664  idx = instr_regarray_idx_imm(p, ip);
3665  src = instr_regarray_src_hbo(t, ip);
3666  regarray[idx] = src;
3667 }
3668 
3669 static inline void
3670 __instr_regwr_rii_exec(struct rte_swx_pipeline *p,
3671  struct thread *t __rte_unused,
3672  const struct instruction *ip)
3673 {
3674  uint64_t *regarray, idx, src;
3675 
3676  TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
3677 
3678  regarray = instr_regarray_regarray(p, ip);
3679  idx = instr_regarray_idx_imm(p, ip);
3680  src = ip->regarray.dstsrc_val;
3681  regarray[idx] = src;
3682 }
3683 
3684 static inline void
3685 __instr_regadd_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3686 {
3687  uint64_t *regarray, idx, src;
3688 
3689  TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
3690 
3691  regarray = instr_regarray_regarray(p, ip);
3692  idx = instr_regarray_idx_nbo(p, t, ip);
3693  src = instr_regarray_src_nbo(t, ip);
3694  regarray[idx] += src;
3695 }
3696 
3697 static inline void
3698 __instr_regadd_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3699 {
3700  uint64_t *regarray, idx, src;
3701 
3702  TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
3703 
3704  regarray = instr_regarray_regarray(p, ip);
3705  idx = instr_regarray_idx_nbo(p, t, ip);
3706  src = instr_regarray_src_hbo(t, ip);
3707  regarray[idx] += src;
3708 }
3709 
3710 static inline void
3711 __instr_regadd_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3712 {
3713  uint64_t *regarray, idx, src;
3714 
3715  TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
3716 
3717  regarray = instr_regarray_regarray(p, ip);
3718  idx = instr_regarray_idx_hbo(p, t, ip);
3719  src = instr_regarray_src_nbo(t, ip);
3720  regarray[idx] += src;
3721 }
3722 
3723 static inline void
3724 __instr_regadd_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3725 {
3726  uint64_t *regarray, idx, src;
3727 
3728  TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
3729 
3730  regarray = instr_regarray_regarray(p, ip);
3731  idx = instr_regarray_idx_hbo(p, t, ip);
3732  src = instr_regarray_src_hbo(t, ip);
3733  regarray[idx] += src;
3734 }
3735 
3736 static inline void
3737 __instr_regadd_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3738 {
3739  uint64_t *regarray, idx, src;
3740 
3741  TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
3742 
3743  regarray = instr_regarray_regarray(p, ip);
3744  idx = instr_regarray_idx_nbo(p, t, ip);
3745  src = ip->regarray.dstsrc_val;
3746  regarray[idx] += src;
3747 }
3748 
3749 static inline void
3750 __instr_regadd_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3751 {
3752  uint64_t *regarray, idx, src;
3753 
3754  TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
3755 
3756  regarray = instr_regarray_regarray(p, ip);
3757  idx = instr_regarray_idx_hbo(p, t, ip);
3758  src = ip->regarray.dstsrc_val;
3759  regarray[idx] += src;
3760 }
3761 
3762 static inline void
3763 __instr_regadd_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3764 {
3765  uint64_t *regarray, idx, src;
3766 
3767  TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
3768 
3769  regarray = instr_regarray_regarray(p, ip);
3770  idx = instr_regarray_idx_imm(p, ip);
3771  src = instr_regarray_src_nbo(t, ip);
3772  regarray[idx] += src;
3773 }
3774 
3775 static inline void
3776 __instr_regadd_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3777 {
3778  uint64_t *regarray, idx, src;
3779 
3780  TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
3781 
3782  regarray = instr_regarray_regarray(p, ip);
3783  idx = instr_regarray_idx_imm(p, ip);
3784  src = instr_regarray_src_hbo(t, ip);
3785  regarray[idx] += src;
3786 }
3787 
3788 static inline void
3789 __instr_regadd_rii_exec(struct rte_swx_pipeline *p,
3790  struct thread *t __rte_unused,
3791  const struct instruction *ip)
3792 {
3793  uint64_t *regarray, idx, src;
3794 
3795  TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
3796 
3797  regarray = instr_regarray_regarray(p, ip);
3798  idx = instr_regarray_idx_imm(p, ip);
3799  src = ip->regarray.dstsrc_val;
3800  regarray[idx] += src;
3801 }
3802 
3803 /*
3804  * metarray.
3805  */
3806 static inline struct meter *
3807 instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3808 {
3809  struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
3810 
3811  uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
3812  uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
3813  uint64_t idx64 = *idx64_ptr;
3814  uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
3815  uint64_t idx = idx64 & idx64_mask & r->size_mask;
3816 
3817  return &r->metarray[idx];
3818 }
3819 
3820 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3821 
3822 static inline struct meter *
3823 instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3824 {
3825  struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
3826 
3827  uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
3828  uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
3829  uint64_t idx64 = *idx64_ptr;
3830  uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
3831 
3832  return &r->metarray[idx];
3833 }
3834 
3835 #else
3836 
3837 #define instr_meter_idx_nbo instr_meter_idx_hbo
3838 
3839 #endif
3840 
3841 static inline struct meter *
3842 instr_meter_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip)
3843 {
3844  struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
3845 
3846  uint64_t idx = ip->meter.idx_val & r->size_mask;
3847 
3848  return &r->metarray[idx];
3849 }
3850 
3851 static inline uint32_t
3852 instr_meter_length_hbo(struct thread *t, const struct instruction *ip)
3853 {
3854  uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
3855  uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
3856  uint64_t src64 = *src64_ptr;
3857  uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
3858  uint64_t src = src64 & src64_mask;
3859 
3860  return (uint32_t)src;
3861 }
3862 
3863 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3864 
3865 static inline uint32_t
3866 instr_meter_length_nbo(struct thread *t, const struct instruction *ip)
3867 {
3868  uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
3869  uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
3870  uint64_t src64 = *src64_ptr;
3871  uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
3872 
3873  return (uint32_t)src;
3874 }
3875 
3876 #else
3877 
3878 #define instr_meter_length_nbo instr_meter_length_hbo
3879 
3880 #endif
3881 
3882 static inline enum rte_color
3883 instr_meter_color_in_hbo(struct thread *t, const struct instruction *ip)
3884 {
3885  uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
3886  uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
3887  uint64_t src64 = *src64_ptr;
3888  uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
3889  uint64_t src = src64 & src64_mask;
3890 
3891  return (enum rte_color)src;
3892 }
3893 
3894 static inline void
3895 instr_meter_color_out_hbo_set(struct thread *t,
3896  const struct instruction *ip,
3897  enum rte_color color_out)
3898 {
3899  uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
3900  uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
3901  uint64_t dst64 = *dst64_ptr;
3902  uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
3903 
3904  uint64_t src = (uint64_t)color_out;
3905 
3906  *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
3907 }
3908 
3909 static inline void
3910 __instr_metprefetch_h_exec(struct rte_swx_pipeline *p,
3911  struct thread *t,
3912  const struct instruction *ip)
3913 {
3914  struct meter *m;
3915 
3916  TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
3917 
3918  m = instr_meter_idx_nbo(p, t, ip);
3919  rte_prefetch0(m);
3920 }
3921 
3922 static inline void
3923 __instr_metprefetch_m_exec(struct rte_swx_pipeline *p,
3924  struct thread *t,
3925  const struct instruction *ip)
3926 {
3927  struct meter *m;
3928 
3929  TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
3930 
3931  m = instr_meter_idx_hbo(p, t, ip);
3932  rte_prefetch0(m);
3933 }
3934 
3935 static inline void
3936 __instr_metprefetch_i_exec(struct rte_swx_pipeline *p,
3937  struct thread *t __rte_unused,
3938  const struct instruction *ip)
3939 {
3940  struct meter *m;
3941 
3942  TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
3943 
3944  m = instr_meter_idx_imm(p, ip);
3945  rte_prefetch0(m);
3946 }
3947 
3948 static inline void
3949 __instr_meter_hhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3950 {
3951  struct meter *m;
3952  uint64_t time, n_pkts, n_bytes;
3953  uint32_t length;
3954  enum rte_color color_in, color_out;
3955 
3956  TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
3957 
3958  m = instr_meter_idx_nbo(p, t, ip);
3959  rte_prefetch0(m->n_pkts);
3960  time = rte_get_tsc_cycles();
3961  length = instr_meter_length_nbo(t, ip);
3962  color_in = instr_meter_color_in_hbo(t, ip);
3963 
3964  color_out = rte_meter_trtcm_color_aware_check(&m->m,
3965  &m->profile->profile,
3966  time,
3967  length,
3968  color_in);
3969 
3970  color_out &= m->color_mask;
3971 
3972  n_pkts = m->n_pkts[color_out];
3973  n_bytes = m->n_bytes[color_out];
3974 
3975  instr_meter_color_out_hbo_set(t, ip, color_out);
3976 
3977  m->n_pkts[color_out] = n_pkts + 1;
3978  m->n_bytes[color_out] = n_bytes + length;
3979 }
3980 
3981 static inline void
3982 __instr_meter_hhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3983 {
3984  struct meter *m;
3985  uint64_t time, n_pkts, n_bytes;
3986  uint32_t length;
3987  enum rte_color color_in, color_out;
3988 
3989  TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
3990 
3991  m = instr_meter_idx_nbo(p, t, ip);
3992  rte_prefetch0(m->n_pkts);
3993  time = rte_get_tsc_cycles();
3994  length = instr_meter_length_nbo(t, ip);
3995  color_in = (enum rte_color)ip->meter.color_in_val;
3996 
3997  color_out = rte_meter_trtcm_color_aware_check(&m->m,
3998  &m->profile->profile,
3999  time,
4000  length,
4001  color_in);
4002 
4003  color_out &= m->color_mask;
4004 
4005  n_pkts = m->n_pkts[color_out];
4006  n_bytes = m->n_bytes[color_out];
4007 
4008  instr_meter_color_out_hbo_set(t, ip, color_out);
4009 
4010  m->n_pkts[color_out] = n_pkts + 1;
4011  m->n_bytes[color_out] = n_bytes + length;
4012 }
4013 
4014 static inline void
4015 __instr_meter_hmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4016 {
4017  struct meter *m;
4018  uint64_t time, n_pkts, n_bytes;
4019  uint32_t length;
4020  enum rte_color color_in, color_out;
4021 
4022  TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
4023 
4024  m = instr_meter_idx_nbo(p, t, ip);
4025  rte_prefetch0(m->n_pkts);
4026  time = rte_get_tsc_cycles();
4027  length = instr_meter_length_hbo(t, ip);
4028  color_in = instr_meter_color_in_hbo(t, ip);
4029 
4030  color_out = rte_meter_trtcm_color_aware_check(&m->m,
4031  &m->profile->profile,
4032  time,
4033  length,
4034  color_in);
4035 
4036  color_out &= m->color_mask;
4037 
4038  n_pkts = m->n_pkts[color_out];
4039  n_bytes = m->n_bytes[color_out];
4040 
4041  instr_meter_color_out_hbo_set(t, ip, color_out);
4042 
4043  m->n_pkts[color_out] = n_pkts + 1;
4044  m->n_bytes[color_out] = n_bytes + length;
4045 }
4046 
4047 static inline void
4048 __instr_meter_hmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4049 {
4050  struct meter *m;
4051  uint64_t time, n_pkts, n_bytes;
4052  uint32_t length;
4053  enum rte_color color_in, color_out;
4054 
4055  TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
4056 
4057  m = instr_meter_idx_nbo(p, t, ip);
4058  rte_prefetch0(m->n_pkts);
4059  time = rte_get_tsc_cycles();
4060  length = instr_meter_length_hbo(t, ip);
4061  color_in = (enum rte_color)ip->meter.color_in_val;
4062 
4063  color_out = rte_meter_trtcm_color_aware_check(&m->m,
4064  &m->profile->profile,
4065  time,
4066  length,
4067  color_in);
4068 
4069  color_out &= m->color_mask;
4070 
4071  n_pkts = m->n_pkts[color_out];
4072  n_bytes = m->n_bytes[color_out];
4073 
4074  instr_meter_color_out_hbo_set(t, ip, color_out);
4075 
4076  m->n_pkts[color_out] = n_pkts + 1;
4077  m->n_bytes[color_out] = n_bytes + length;
4078 }
4079 
4080 static inline void
4081 __instr_meter_mhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4082 {
4083  struct meter *m;
4084  uint64_t time, n_pkts, n_bytes;
4085  uint32_t length;
4086  enum rte_color color_in, color_out;
4087 
4088  TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
4089 
4090  m = instr_meter_idx_hbo(p, t, ip);
4091  rte_prefetch0(m->n_pkts);
4092  time = rte_get_tsc_cycles();
4093  length = instr_meter_length_nbo(t, ip);
4094  color_in = instr_meter_color_in_hbo(t, ip);
4095 
4096  color_out = rte_meter_trtcm_color_aware_check(&m->m,
4097  &m->profile->profile,
4098  time,
4099  length,
4100  color_in);
4101 
4102  color_out &= m->color_mask;
4103 
4104  n_pkts = m->n_pkts[color_out];
4105  n_bytes = m->n_bytes[color_out];
4106 
4107  instr_meter_color_out_hbo_set(t, ip, color_out);
4108 
4109  m->n_pkts[color_out] = n_pkts + 1;
4110  m->n_bytes[color_out] = n_bytes + length;
4111 }
4112 
4113 static inline void
4114 __instr_meter_mhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4115 {
4116  struct meter *m;
4117  uint64_t time, n_pkts, n_bytes;
4118  uint32_t length;
4119  enum rte_color color_in, color_out;
4120 
4121  TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
4122 
4123  m = instr_meter_idx_hbo(p, t, ip);
4124  rte_prefetch0(m->n_pkts);
4125  time = rte_get_tsc_cycles();
4126  length = instr_meter_length_nbo(t, ip);
4127  color_in = (enum rte_color)ip->meter.color_in_val;
4128 
4129  color_out = rte_meter_trtcm_color_aware_check(&m->m,
4130  &m->profile->profile,
4131  time,
4132  length,
4133  color_in);
4134 
4135  color_out &= m->color_mask;
4136 
4137  n_pkts = m->n_pkts[color_out];
4138  n_bytes = m->n_bytes[color_out];
4139 
4140  instr_meter_color_out_hbo_set(t, ip, color_out);
4141 
4142  m->n_pkts[color_out] = n_pkts + 1;
4143  m->n_bytes[color_out] = n_bytes + length;
4144 }
4145 
4146 static inline void
4147 __instr_meter_mmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4148 {
4149  struct meter *m;
4150  uint64_t time, n_pkts, n_bytes;
4151  uint32_t length;
4152  enum rte_color color_in, color_out;
4153 
4154  TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
4155 
4156  m = instr_meter_idx_hbo(p, t, ip);
4157  rte_prefetch0(m->n_pkts);
4158  time = rte_get_tsc_cycles();
4159  length = instr_meter_length_hbo(t, ip);
4160  color_in = instr_meter_color_in_hbo(t, ip);
4161 
4162  color_out = rte_meter_trtcm_color_aware_check(&m->m,
4163  &m->profile->profile,
4164  time,
4165  length,
4166  color_in);
4167 
4168  color_out &= m->color_mask;
4169 
4170  n_pkts = m->n_pkts[color_out];
4171  n_bytes = m->n_bytes[color_out];
4172 
4173  instr_meter_color_out_hbo_set(t, ip, color_out);
4174 
4175  m->n_pkts[color_out] = n_pkts + 1;
4176  m->n_bytes[color_out] = n_bytes + length;
4177 }
4178 
4179 static inline void
4180 __instr_meter_mmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4181 {
4182  struct meter *m;
4183  uint64_t time, n_pkts, n_bytes;
4184  uint32_t length;
4185  enum rte_color color_in, color_out;
4186 
4187  TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
4188 
4189  m = instr_meter_idx_hbo(p, t, ip);
4190  rte_prefetch0(m->n_pkts);
4191  time = rte_get_tsc_cycles();
4192  length = instr_meter_length_hbo(t, ip);
4193  color_in = (enum rte_color)ip->meter.color_in_val;
4194 
4195  color_out = rte_meter_trtcm_color_aware_check(&m->m,
4196  &m->profile->profile,
4197  time,
4198  length,
4199  color_in);
4200 
4201  color_out &= m->color_mask;
4202 
4203  n_pkts = m->n_pkts[color_out];
4204  n_bytes = m->n_bytes[color_out];
4205 
4206  instr_meter_color_out_hbo_set(t, ip, color_out);
4207 
4208  m->n_pkts[color_out] = n_pkts + 1;
4209  m->n_bytes[color_out] = n_bytes + length;
4210 }
4211 
4212 static inline void
4213 __instr_meter_ihm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4214 {
4215  struct meter *m;
4216  uint64_t time, n_pkts, n_bytes;
4217  uint32_t length;
4218  enum rte_color color_in, color_out;
4219 
4220  TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
4221 
4222  m = instr_meter_idx_imm(p, ip);
4223  rte_prefetch0(m->n_pkts);
4224  time = rte_get_tsc_cycles();
4225  length = instr_meter_length_nbo(t, ip);
4226  color_in = instr_meter_color_in_hbo(t, ip);
4227 
4228  color_out = rte_meter_trtcm_color_aware_check(&m->m,
4229  &m->profile->profile,
4230  time,
4231  length,
4232  color_in);
4233 
4234  color_out &= m->color_mask;
4235 
4236  n_pkts = m->n_pkts[color_out];
4237  n_bytes = m->n_bytes[color_out];
4238 
4239  instr_meter_color_out_hbo_set(t, ip, color_out);
4240 
4241  m->n_pkts[color_out] = n_pkts + 1;
4242  m->n_bytes[color_out] = n_bytes + length;
4243 }
4244 
4245 static inline void
4246 __instr_meter_ihi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4247 {
4248  struct meter *m;
4249  uint64_t time, n_pkts, n_bytes;
4250  uint32_t length;
4251  enum rte_color color_in, color_out;
4252 
4253  TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
4254 
4255  m = instr_meter_idx_imm(p, ip);
4256  rte_prefetch0(m->n_pkts);
4257  time = rte_get_tsc_cycles();
4258  length = instr_meter_length_nbo(t, ip);
4259  color_in = (enum rte_color)ip->meter.color_in_val;
4260 
4261  color_out = rte_meter_trtcm_color_aware_check(&m->m,
4262  &m->profile->profile,
4263  time,
4264  length,
4265  color_in);
4266 
4267  color_out &= m->color_mask;
4268 
4269  n_pkts = m->n_pkts[color_out];
4270  n_bytes = m->n_bytes[color_out];
4271 
4272  instr_meter_color_out_hbo_set(t, ip, color_out);
4273 
4274  m->n_pkts[color_out] = n_pkts + 1;
4275  m->n_bytes[color_out] = n_bytes + length;
4276 }
4277 
4278 static inline void
4279 __instr_meter_imm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4280 {
4281  struct meter *m;
4282  uint64_t time, n_pkts, n_bytes;
4283  uint32_t length;
4284  enum rte_color color_in, color_out;
4285 
4286  TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
4287 
4288  m = instr_meter_idx_imm(p, ip);
4289  rte_prefetch0(m->n_pkts);
4290  time = rte_get_tsc_cycles();
4291  length = instr_meter_length_hbo(t, ip);
4292  color_in = instr_meter_color_in_hbo(t, ip);
4293 
4294  color_out = rte_meter_trtcm_color_aware_check(&m->m,
4295  &m->profile->profile,
4296  time,
4297  length,
4298  color_in);
4299 
4300  color_out &= m->color_mask;
4301 
4302  n_pkts = m->n_pkts[color_out];
4303  n_bytes = m->n_bytes[color_out];
4304 
4305  instr_meter_color_out_hbo_set(t, ip, color_out);
4306 
4307  m->n_pkts[color_out] = n_pkts + 1;
4308  m->n_bytes[color_out] = n_bytes + length;
4309 }
4310 
4311 static inline void
4312 __instr_meter_imi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4313 {
4314  struct meter *m;
4315  uint64_t time, n_pkts, n_bytes;
4316  uint32_t length;
4317  enum rte_color color_in, color_out;
4318 
4319  TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
4320 
4321  m = instr_meter_idx_imm(p, ip);
4322  rte_prefetch0(m->n_pkts);
4323  time = rte_get_tsc_cycles();
4324  length = instr_meter_length_hbo(t, ip);
4325  color_in = (enum rte_color)ip->meter.color_in_val;
4326 
4327  color_out = rte_meter_trtcm_color_aware_check(&m->m,
4328  &m->profile->profile,
4329  time,
4330  length,
4331  color_in);
4332 
4333  color_out &= m->color_mask;
4334 
4335  n_pkts = m->n_pkts[color_out];
4336  n_bytes = m->n_bytes[color_out];
4337 
4338  instr_meter_color_out_hbo_set(t, ip, color_out);
4339 
4340  m->n_pkts[color_out] = n_pkts + 1;
4341  m->n_bytes[color_out] = n_bytes + length;
4342 }
4343 
4344 #endif
#define __rte_unused
Definition: rte_common.h:120
static uint64_t rte_get_tsc_cycles(void)
static enum rte_color rte_meter_trtcm_color_aware_check(struct rte_meter_trtcm *m, struct rte_meter_trtcm_profile *p, uint64_t time, uint32_t pkt_len, enum rte_color pkt_color)
Definition: rte_meter.h:538
rte_color
Definition: rte_meter.h:35
@ RTE_COLORS
Definition: rte_meter.h:39
static void rte_prefetch0(const volatile void *p)
void(* rte_swx_extern_type_destructor_t)(void *object)
void *(* rte_swx_extern_type_constructor_t)(const char *args)
int(* rte_swx_extern_func_t)(void *mailbox)
int(* rte_swx_extern_type_member_func_t)(void *object, void *mailbox)
uint32_t(* rte_swx_hash_func_t)(const void *key, uint32_t length, uint32_t seed)
#define RTE_SWX_NAME_SIZE
int(* rte_swx_port_in_pkt_rx_t)(void *port, struct rte_swx_pkt *pkt)
Definition: rte_swx_port.h:72
void(* rte_swx_port_out_flush_t)(void *port)
Definition: rte_swx_port.h:184
void(* rte_swx_port_out_pkt_clone_tx_t)(void *port, struct rte_swx_pkt *pkt, uint32_t truncation_length)
Definition: rte_swx_port.h:173
void(* rte_swx_port_out_pkt_tx_t)(void *port, struct rte_swx_pkt *pkt)
Definition: rte_swx_port.h:147
void(* rte_swx_port_out_pkt_fast_clone_tx_t)(void *port, struct rte_swx_pkt *pkt)
Definition: rte_swx_port.h:159
rte_swx_table_match_type
Definition: rte_swx_table.h:25
int(* rte_swx_table_lookup_t)(void *table, void *mailbox, uint8_t **key, uint64_t *action_id, uint8_t **action_data, size_t *entry_id, int *hit)
__rte_experimental uint32_t rte_swx_table_learner_add(void *table, void *mailbox, uint64_t time, uint64_t action_id, uint8_t *action_data, uint32_t key_timeout_id)
__rte_experimental void rte_swx_table_learner_rearm_new(void *table, void *mailbox, uint64_t time, uint32_t key_timeout_id)
__rte_experimental void rte_swx_table_learner_delete(void *table, void *mailbox)
__rte_experimental void rte_swx_table_learner_rearm(void *table, void *mailbox, uint64_t time)
#define RTE_SWX_TABLE_LEARNER_N_KEY_TIMEOUTS_MAX
uint8_t * pkt
Definition: rte_swx_port.h:26
uint32_t offset
Definition: rte_swx_port.h:29
uint32_t length
Definition: rte_swx_port.h:32