Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0-or-later
2 : /**
3 : * bfd.c: BFD handling routines
4 : *
5 : * @copyright Copyright (C) 2015 Cumulus Networks, Inc.
6 : */
7 :
8 : #include <zebra.h>
9 :
10 : #include "command.h"
11 : #include "memory.h"
12 : #include "prefix.h"
13 : #include "frrevent.h"
14 : #include "stream.h"
15 : #include "vrf.h"
16 : #include "zclient.h"
17 : #include "libfrr.h"
18 : #include "table.h"
19 : #include "vty.h"
20 : #include "bfd.h"
21 :
22 12 : DEFINE_MTYPE_STATIC(LIB, BFD_INFO, "BFD info");
23 12 : DEFINE_MTYPE_STATIC(LIB, BFD_SOURCE, "BFD source cache");
24 :
25 : /**
26 : * BFD protocol integration configuration.
27 : */
28 :
29 : /** Events definitions. */
30 : enum bfd_session_event {
31 : /** Remove the BFD session configuration. */
32 : BSE_UNINSTALL,
33 : /** Install the BFD session configuration. */
34 : BSE_INSTALL,
35 : };
36 :
37 : /**
38 : * BFD source selection result cache.
39 : *
40 : * This structure will keep track of the result based on the destination
41 : * prefix. When the result changes all related BFD sessions with automatic
42 : * source will be updated.
43 : */
44 : struct bfd_source_cache {
45 : /** Address VRF belongs. */
46 : vrf_id_t vrf_id;
47 : /** Destination network address. */
48 : struct prefix address;
49 : /** Source selected. */
50 : struct prefix source;
51 : /** Is the source address valid? */
52 : bool valid;
53 : /** BFD sessions using this. */
54 : size_t refcount;
55 :
56 : SLIST_ENTRY(bfd_source_cache) entry;
57 : };
58 : SLIST_HEAD(bfd_source_list, bfd_source_cache);
59 :
60 : /**
61 : * Data structure to do the necessary tricks to hide the BFD protocol
62 : * integration internals.
63 : */
64 : struct bfd_session_params {
65 : /** Contains the session parameters and more. */
66 : struct bfd_session_arg args;
67 : /** Contains the session state. */
68 : struct bfd_session_status bss;
69 : /** Protocol implementation status update callback. */
70 : bsp_status_update updatecb;
71 : /** Protocol implementation custom data pointer. */
72 : void *arg;
73 :
74 : /**
75 : * Next event.
76 : *
77 : * This variable controls what action to execute when the command batch
78 : * finishes. Normally we'd use `event_add_event` value, however since
79 : * that function is going to be called multiple times and the value
80 : * might be different we'll use this variable to keep track of it.
81 : */
82 : enum bfd_session_event lastev;
83 : /**
84 : * BFD session configuration event.
85 : *
86 : * Multiple actions might be asked during a command batch (either via
87 : * configuration load or northbound batch), so we'll use this to
88 : * install/uninstall the BFD session parameters only once.
89 : */
90 : struct event *installev;
91 :
92 : /** BFD session installation state. */
93 : bool installed;
94 :
95 : /** Automatic source selection. */
96 : bool auto_source;
97 : /** Currently selected source. */
98 : struct bfd_source_cache *source_cache;
99 :
100 : /** Global BFD paramaters list. */
101 : TAILQ_ENTRY(bfd_session_params) entry;
102 : };
103 :
104 : struct bfd_sessions_global {
105 : /**
106 : * Global BFD session parameters list for (re)installation and update
107 : * without code duplication among daemons.
108 : */
109 : TAILQ_HEAD(bsplist, bfd_session_params) bsplist;
110 : /** BFD automatic source selection cache. */
111 : struct bfd_source_list source_list;
112 :
113 : /** Pointer to FRR's event manager. */
114 : struct event_loop *tm;
115 : /** Pointer to zebra client data structure. */
116 : struct zclient *zc;
117 :
118 : /** Debugging state. */
119 : bool debugging;
120 : /** Is shutting down? */
121 : bool shutting_down;
122 : };
123 :
124 : /** Global configuration variable. */
125 : static struct bfd_sessions_global bsglobal;
126 :
127 : /** Global empty address for IPv4/IPv6. */
128 : static const struct in6_addr i6a_zero;
129 :
130 : /*
131 : * Prototypes
132 : */
133 :
134 : static void bfd_source_cache_get(struct bfd_session_params *session);
135 : static void bfd_source_cache_put(struct bfd_session_params *session);
136 :
137 : /*
138 : * bfd_get_peer_info - Extract the Peer information for which the BFD session
139 : * went down from the message sent from Zebra to clients.
140 : */
141 0 : static struct interface *bfd_get_peer_info(struct stream *s, struct prefix *dp,
142 : struct prefix *sp, int *status,
143 : int *remote_cbit, vrf_id_t vrf_id)
144 : {
145 0 : unsigned int ifindex;
146 0 : struct interface *ifp = NULL;
147 0 : int plen;
148 0 : int local_remote_cbit;
149 :
150 : /*
151 : * If the ifindex lookup fails the
152 : * rest of the data in the stream is
153 : * not read. All examples of this function
154 : * call immediately use the dp->family which
155 : * is not good. Ensure we are not using
156 : * random data
157 : */
158 0 : memset(dp, 0, sizeof(*dp));
159 0 : memset(sp, 0, sizeof(*sp));
160 :
161 : /* Get interface index. */
162 0 : STREAM_GETL(s, ifindex);
163 :
164 : /* Lookup index. */
165 0 : if (ifindex != 0) {
166 0 : ifp = if_lookup_by_index(ifindex, vrf_id);
167 0 : if (ifp == NULL) {
168 0 : if (bsglobal.debugging)
169 0 : zlog_debug(
170 : "%s: Can't find interface by ifindex: %d ",
171 : __func__, ifindex);
172 0 : return NULL;
173 : }
174 : }
175 :
176 : /* Fetch destination address. */
177 0 : STREAM_GETC(s, dp->family);
178 :
179 0 : plen = prefix_blen(dp);
180 0 : STREAM_GET(&dp->u.prefix, s, plen);
181 0 : STREAM_GETC(s, dp->prefixlen);
182 :
183 : /* Get BFD status. */
184 0 : STREAM_GETL(s, (*status));
185 :
186 0 : STREAM_GETC(s, sp->family);
187 :
188 0 : plen = prefix_blen(sp);
189 0 : STREAM_GET(&sp->u.prefix, s, plen);
190 0 : STREAM_GETC(s, sp->prefixlen);
191 :
192 0 : STREAM_GETC(s, local_remote_cbit);
193 0 : if (remote_cbit)
194 0 : *remote_cbit = local_remote_cbit;
195 : return ifp;
196 :
197 0 : stream_failure:
198 : /*
199 : * Clean dp and sp because caller
200 : * will immediately check them valid or not
201 : */
202 0 : memset(dp, 0, sizeof(*dp));
203 0 : memset(sp, 0, sizeof(*sp));
204 0 : return NULL;
205 : }
206 :
207 : /*
208 : * bfd_get_status_str - Convert BFD status to a display string.
209 : */
210 0 : const char *bfd_get_status_str(int status)
211 : {
212 0 : switch (status) {
213 : case BFD_STATUS_DOWN:
214 : return "Down";
215 0 : case BFD_STATUS_UP:
216 0 : return "Up";
217 0 : case BFD_STATUS_ADMIN_DOWN:
218 0 : return "Admin Down";
219 0 : case BFD_STATUS_UNKNOWN:
220 : default:
221 0 : return "Unknown";
222 : }
223 : }
224 :
225 : /*
226 : * bfd_last_update - Calculate the last BFD update time and convert it
227 : * into a dd:hh:mm:ss display format.
228 : */
229 0 : static void bfd_last_update(time_t last_update, char *buf, size_t len)
230 : {
231 0 : time_t curr;
232 0 : time_t diff;
233 0 : struct tm tm;
234 0 : struct timeval tv;
235 :
236 : /* If no BFD status update has ever been received, print `never'. */
237 0 : if (last_update == 0) {
238 0 : snprintf(buf, len, "never");
239 0 : return;
240 : }
241 :
242 : /* Get current time. */
243 0 : monotime(&tv);
244 0 : curr = tv.tv_sec;
245 0 : diff = curr - last_update;
246 0 : gmtime_r(&diff, &tm);
247 :
248 0 : snprintf(buf, len, "%d:%02d:%02d:%02d", tm.tm_yday, tm.tm_hour,
249 : tm.tm_min, tm.tm_sec);
250 : }
251 :
252 : /*
253 : * bfd_client_sendmsg - Format and send a client register
254 : * command to Zebra to be forwarded to BFD
255 : */
256 4 : void bfd_client_sendmsg(struct zclient *zclient, int command,
257 : vrf_id_t vrf_id)
258 : {
259 4 : struct stream *s;
260 4 : enum zclient_send_status ret;
261 :
262 : /* Check socket. */
263 4 : if (!zclient || zclient->sock < 0) {
264 2 : if (bsglobal.debugging)
265 0 : zlog_debug(
266 : "%s: Can't send BFD client register, Zebra client not established",
267 : __func__);
268 2 : return;
269 : }
270 :
271 2 : s = zclient->obuf;
272 2 : stream_reset(s);
273 2 : zclient_create_header(s, command, vrf_id);
274 :
275 2 : stream_putl(s, getpid());
276 :
277 2 : stream_putw_at(s, 0, stream_get_endp(s));
278 :
279 2 : ret = zclient_send_message(zclient);
280 :
281 2 : if (ret == ZCLIENT_SEND_FAILURE) {
282 0 : if (bsglobal.debugging)
283 0 : zlog_debug(
284 : "%s: %ld: zclient_send_message() failed",
285 : __func__, (long)getpid());
286 0 : return;
287 : }
288 :
289 : return;
290 : }
291 :
292 0 : int zclient_bfd_command(struct zclient *zc, struct bfd_session_arg *args)
293 : {
294 0 : struct stream *s;
295 0 : size_t addrlen;
296 :
297 : /* Individual reg/dereg messages are suppressed during shutdown. */
298 0 : if (bsglobal.shutting_down) {
299 0 : if (bsglobal.debugging)
300 0 : zlog_debug(
301 : "%s: Suppressing BFD peer reg/dereg messages",
302 : __func__);
303 0 : return -1;
304 : }
305 :
306 : /* Check socket. */
307 0 : if (!zc || zc->sock < 0) {
308 0 : if (bsglobal.debugging)
309 0 : zlog_debug("%s: zclient unavailable", __func__);
310 0 : return -1;
311 : }
312 :
313 0 : s = zc->obuf;
314 0 : stream_reset(s);
315 :
316 : /* Create new message. */
317 0 : zclient_create_header(s, args->command, args->vrf_id);
318 0 : stream_putl(s, getpid());
319 :
320 : /* Encode destination address. */
321 0 : stream_putw(s, args->family);
322 0 : addrlen = (args->family == AF_INET) ? sizeof(struct in_addr)
323 0 : : sizeof(struct in6_addr);
324 0 : stream_put(s, &args->dst, addrlen);
325 :
326 : /*
327 : * For more BFD integration protocol details, see function
328 : * `_ptm_msg_read` in `bfdd/ptm_adapter.c`.
329 : */
330 : #if HAVE_BFDD > 0
331 : /* Session timers. */
332 0 : stream_putl(s, args->min_rx);
333 0 : stream_putl(s, args->min_tx);
334 0 : stream_putc(s, args->detection_multiplier);
335 :
336 : /* Is multi hop? */
337 0 : stream_putc(s, args->mhop != 0);
338 :
339 : /* Source address. */
340 0 : stream_putw(s, args->family);
341 0 : stream_put(s, &args->src, addrlen);
342 :
343 : /* Send the expected hops. */
344 0 : stream_putc(s, args->hops);
345 :
346 : /* Send interface name if any. */
347 0 : if (args->mhop) {
348 : /* Don't send interface. */
349 0 : stream_putc(s, 0);
350 0 : if (bsglobal.debugging && args->ifnamelen)
351 0 : zlog_debug("%s: multi hop is configured, not sending interface",
352 : __func__);
353 : } else {
354 0 : stream_putc(s, args->ifnamelen);
355 0 : if (args->ifnamelen)
356 0 : stream_put(s, args->ifname, args->ifnamelen);
357 : }
358 :
359 : /* Send the C bit indicator. */
360 0 : stream_putc(s, args->cbit);
361 :
362 : /* Send profile name if any. */
363 0 : stream_putc(s, args->profilelen);
364 0 : if (args->profilelen)
365 0 : stream_put(s, args->profile, args->profilelen);
366 : #else /* PTM BFD */
367 : /* Encode timers if this is a registration message. */
368 : if (args->command != ZEBRA_BFD_DEST_DEREGISTER) {
369 : stream_putl(s, args->min_rx);
370 : stream_putl(s, args->min_tx);
371 : stream_putc(s, args->detection_multiplier);
372 : }
373 :
374 : if (args->mhop) {
375 : /* Multi hop indicator. */
376 : stream_putc(s, 1);
377 :
378 : /* Multi hop always sends the source address. */
379 : stream_putw(s, args->family);
380 : stream_put(s, &args->src, addrlen);
381 :
382 : /* Send the expected hops. */
383 : stream_putc(s, args->hops);
384 : } else {
385 : /* Multi hop indicator. */
386 : stream_putc(s, 0);
387 :
388 : /* Single hop only sends the source address when IPv6. */
389 : if (args->family == AF_INET6) {
390 : stream_putw(s, args->family);
391 : stream_put(s, &args->src, addrlen);
392 : }
393 :
394 : /* Send interface name if any. */
395 : stream_putc(s, args->ifnamelen);
396 : if (args->ifnamelen)
397 : stream_put(s, args->ifname, args->ifnamelen);
398 : }
399 :
400 : /* Send the C bit indicator. */
401 : stream_putc(s, args->cbit);
402 : #endif /* HAVE_BFDD */
403 :
404 : /* Finish the message by writing the size. */
405 0 : stream_putw_at(s, 0, stream_get_endp(s));
406 :
407 : /* Send message to zebra. */
408 0 : if (zclient_send_message(zc) == ZCLIENT_SEND_FAILURE) {
409 0 : if (bsglobal.debugging)
410 0 : zlog_debug("%s: zclient_send_message failed", __func__);
411 0 : return -1;
412 : }
413 :
414 : return 0;
415 : }
416 :
417 0 : struct bfd_session_params *bfd_sess_new(bsp_status_update updatecb, void *arg)
418 : {
419 0 : struct bfd_session_params *bsp;
420 :
421 0 : bsp = XCALLOC(MTYPE_BFD_INFO, sizeof(*bsp));
422 :
423 : /* Save application data. */
424 0 : bsp->updatecb = updatecb;
425 0 : bsp->arg = arg;
426 :
427 : /* Set defaults. */
428 0 : bsp->args.detection_multiplier = BFD_DEF_DETECT_MULT;
429 0 : bsp->args.hops = 1;
430 0 : bsp->args.min_rx = BFD_DEF_MIN_RX;
431 0 : bsp->args.min_tx = BFD_DEF_MIN_TX;
432 0 : bsp->args.vrf_id = VRF_DEFAULT;
433 :
434 : /* Register in global list. */
435 0 : TAILQ_INSERT_TAIL(&bsglobal.bsplist, bsp, entry);
436 :
437 0 : return bsp;
438 : }
439 :
440 0 : static bool _bfd_sess_valid(const struct bfd_session_params *bsp)
441 : {
442 : /* Peer/local address not configured. */
443 0 : if (bsp->args.family == 0)
444 : return false;
445 :
446 : /* Address configured but invalid. */
447 0 : if (bsp->args.family != AF_INET && bsp->args.family != AF_INET6) {
448 0 : if (bsglobal.debugging)
449 0 : zlog_debug("%s: invalid session family: %d", __func__,
450 : bsp->args.family);
451 0 : return false;
452 : }
453 :
454 : /* Invalid address. */
455 0 : if (memcmp(&bsp->args.dst, &i6a_zero, sizeof(i6a_zero)) == 0) {
456 0 : if (bsglobal.debugging) {
457 0 : if (bsp->args.family == AF_INET)
458 0 : zlog_debug("%s: invalid address: %pI4",
459 : __func__,
460 : (struct in_addr *)&bsp->args.dst);
461 : else
462 0 : zlog_debug("%s: invalid address: %pI6",
463 : __func__, &bsp->args.dst);
464 : }
465 0 : return false;
466 : }
467 :
468 : /* Multi hop requires local address. */
469 0 : if (bsp->args.mhop
470 0 : && memcmp(&i6a_zero, &bsp->args.src, sizeof(i6a_zero)) == 0) {
471 0 : if (bsglobal.debugging)
472 0 : zlog_debug(
473 : "%s: multi hop but no local address provided",
474 : __func__);
475 0 : return false;
476 : }
477 :
478 : /* Check VRF ID. */
479 0 : if (bsp->args.vrf_id == VRF_UNKNOWN) {
480 0 : if (bsglobal.debugging)
481 0 : zlog_debug("%s: asked for unknown VRF", __func__);
482 0 : return false;
483 : }
484 :
485 : return true;
486 : }
487 :
488 0 : static void _bfd_sess_send(struct event *t)
489 : {
490 0 : struct bfd_session_params *bsp = EVENT_ARG(t);
491 0 : int rv;
492 :
493 : /* Validate configuration before trying to send bogus data. */
494 0 : if (!_bfd_sess_valid(bsp))
495 : return;
496 :
497 0 : if (bsp->lastev == BSE_INSTALL) {
498 0 : bsp->args.command = bsp->installed ? ZEBRA_BFD_DEST_UPDATE
499 0 : : ZEBRA_BFD_DEST_REGISTER;
500 : } else
501 0 : bsp->args.command = ZEBRA_BFD_DEST_DEREGISTER;
502 :
503 : /* If not installed and asked for uninstall, do nothing. */
504 0 : if (!bsp->installed && bsp->args.command == ZEBRA_BFD_DEST_DEREGISTER)
505 : return;
506 :
507 0 : rv = zclient_bfd_command(bsglobal.zc, &bsp->args);
508 : /* Command was sent successfully. */
509 0 : if (rv == 0) {
510 : /* Update installation status. */
511 0 : if (bsp->args.command == ZEBRA_BFD_DEST_DEREGISTER)
512 0 : bsp->installed = false;
513 0 : else if (bsp->args.command == ZEBRA_BFD_DEST_REGISTER)
514 0 : bsp->installed = true;
515 : } else {
516 0 : struct ipaddr src, dst;
517 :
518 0 : src.ipa_type = bsp->args.family;
519 0 : src.ipaddr_v6 = bsp->args.src;
520 0 : dst.ipa_type = bsp->args.family;
521 0 : dst.ipaddr_v6 = bsp->args.dst;
522 :
523 0 : zlog_err(
524 : "%s: BFD session %pIA -> %pIA interface %s VRF %s(%u) was not %s",
525 : __func__, &src, &dst,
526 : bsp->args.ifnamelen ? bsp->args.ifname : "*",
527 : vrf_id_to_name(bsp->args.vrf_id), bsp->args.vrf_id,
528 : bsp->lastev == BSE_INSTALL ? "installed"
529 : : "uninstalled");
530 : }
531 : }
532 :
533 0 : static void _bfd_sess_remove(struct bfd_session_params *bsp)
534 : {
535 : /* Cancel any pending installation request. */
536 0 : EVENT_OFF(bsp->installev);
537 :
538 : /* Not installed, nothing to do. */
539 0 : if (!bsp->installed)
540 : return;
541 :
542 : /* Send request to remove any session. */
543 0 : bsp->lastev = BSE_UNINSTALL;
544 0 : event_execute(bsglobal.tm, _bfd_sess_send, bsp, 0, NULL);
545 : }
546 :
547 0 : void bfd_sess_free(struct bfd_session_params **bsp)
548 : {
549 0 : if (*bsp == NULL)
550 : return;
551 :
552 : /* Remove any installed session. */
553 0 : _bfd_sess_remove(*bsp);
554 :
555 : /* Remove from global list. */
556 0 : TAILQ_REMOVE(&bsglobal.bsplist, (*bsp), entry);
557 :
558 0 : bfd_source_cache_put(*bsp);
559 :
560 : /* Free the memory and point to NULL. */
561 0 : XFREE(MTYPE_BFD_INFO, (*bsp));
562 : }
563 :
564 0 : static bool bfd_sess_address_changed(const struct bfd_session_params *bsp,
565 : uint32_t family,
566 : const struct in6_addr *src,
567 : const struct in6_addr *dst)
568 : {
569 0 : size_t addrlen;
570 :
571 0 : if (bsp->args.family != family)
572 : return true;
573 :
574 0 : addrlen = (family == AF_INET) ? sizeof(struct in_addr)
575 0 : : sizeof(struct in6_addr);
576 0 : if ((src == NULL && memcmp(&bsp->args.src, &i6a_zero, addrlen))
577 0 : || (src && memcmp(src, &bsp->args.src, addrlen))
578 0 : || memcmp(dst, &bsp->args.dst, addrlen))
579 0 : return true;
580 :
581 : return false;
582 : }
583 :
584 0 : void bfd_sess_set_ipv4_addrs(struct bfd_session_params *bsp,
585 : const struct in_addr *src,
586 : const struct in_addr *dst)
587 : {
588 0 : if (!bfd_sess_address_changed(bsp, AF_INET, (struct in6_addr *)src,
589 : (struct in6_addr *)dst))
590 : return;
591 :
592 : /* If already installed, remove the old setting. */
593 0 : _bfd_sess_remove(bsp);
594 : /* Address changed so we must reapply auto source. */
595 0 : bfd_source_cache_put(bsp);
596 :
597 0 : bsp->args.family = AF_INET;
598 :
599 : /* Clean memory, set zero value and avoid static analyser warnings. */
600 0 : memset(&bsp->args.src, 0, sizeof(bsp->args.src));
601 0 : memset(&bsp->args.dst, 0, sizeof(bsp->args.dst));
602 :
603 : /* Copy the equivalent of IPv4 to arguments structure. */
604 0 : if (src)
605 0 : memcpy(&bsp->args.src, src, sizeof(struct in_addr));
606 :
607 0 : assert(dst);
608 0 : memcpy(&bsp->args.dst, dst, sizeof(struct in_addr));
609 :
610 0 : if (bsp->auto_source)
611 0 : bfd_source_cache_get(bsp);
612 : }
613 :
614 0 : void bfd_sess_set_ipv6_addrs(struct bfd_session_params *bsp,
615 : const struct in6_addr *src,
616 : const struct in6_addr *dst)
617 : {
618 0 : if (!bfd_sess_address_changed(bsp, AF_INET6, src, dst))
619 : return;
620 :
621 : /* If already installed, remove the old setting. */
622 0 : _bfd_sess_remove(bsp);
623 : /* Address changed so we must reapply auto source. */
624 0 : bfd_source_cache_put(bsp);
625 :
626 0 : bsp->args.family = AF_INET6;
627 :
628 : /* Clean memory, set zero value and avoid static analyser warnings. */
629 0 : memset(&bsp->args.src, 0, sizeof(bsp->args.src));
630 :
631 0 : if (src)
632 0 : bsp->args.src = *src;
633 :
634 0 : assert(dst);
635 0 : bsp->args.dst = *dst;
636 :
637 0 : if (bsp->auto_source)
638 0 : bfd_source_cache_get(bsp);
639 : }
640 :
641 0 : void bfd_sess_set_interface(struct bfd_session_params *bsp, const char *ifname)
642 : {
643 0 : if ((ifname == NULL && bsp->args.ifnamelen == 0)
644 0 : || (ifname && strcmp(bsp->args.ifname, ifname) == 0))
645 : return;
646 :
647 : /* If already installed, remove the old setting. */
648 0 : _bfd_sess_remove(bsp);
649 :
650 0 : if (ifname == NULL) {
651 0 : bsp->args.ifname[0] = 0;
652 0 : bsp->args.ifnamelen = 0;
653 0 : return;
654 : }
655 :
656 0 : if (strlcpy(bsp->args.ifname, ifname, sizeof(bsp->args.ifname))
657 : > sizeof(bsp->args.ifname))
658 0 : zlog_warn("%s: interface name truncated: %s", __func__, ifname);
659 :
660 0 : bsp->args.ifnamelen = strlen(bsp->args.ifname);
661 : }
662 :
663 0 : void bfd_sess_set_profile(struct bfd_session_params *bsp, const char *profile)
664 : {
665 0 : if (profile == NULL) {
666 0 : bsp->args.profile[0] = 0;
667 0 : bsp->args.profilelen = 0;
668 0 : return;
669 : }
670 :
671 0 : if (strlcpy(bsp->args.profile, profile, sizeof(bsp->args.profile))
672 : > sizeof(bsp->args.profile))
673 0 : zlog_warn("%s: profile name truncated: %s", __func__, profile);
674 :
675 0 : bsp->args.profilelen = strlen(bsp->args.profile);
676 : }
677 :
678 0 : void bfd_sess_set_vrf(struct bfd_session_params *bsp, vrf_id_t vrf_id)
679 : {
680 0 : if (bsp->args.vrf_id == vrf_id)
681 : return;
682 :
683 : /* If already installed, remove the old setting. */
684 0 : _bfd_sess_remove(bsp);
685 : /* Address changed so we must reapply auto source. */
686 0 : bfd_source_cache_put(bsp);
687 :
688 0 : bsp->args.vrf_id = vrf_id;
689 :
690 0 : if (bsp->auto_source)
691 0 : bfd_source_cache_get(bsp);
692 : }
693 :
694 0 : void bfd_sess_set_hop_count(struct bfd_session_params *bsp, uint8_t hops)
695 : {
696 0 : if (bsp->args.hops == hops)
697 : return;
698 :
699 : /* If already installed, remove the old setting. */
700 0 : _bfd_sess_remove(bsp);
701 :
702 0 : bsp->args.hops = hops;
703 0 : bsp->args.mhop = (hops > 1);
704 : }
705 :
706 :
707 0 : void bfd_sess_set_cbit(struct bfd_session_params *bsp, bool enable)
708 : {
709 0 : bsp->args.cbit = enable;
710 0 : }
711 :
712 0 : void bfd_sess_set_timers(struct bfd_session_params *bsp,
713 : uint8_t detection_multiplier, uint32_t min_rx,
714 : uint32_t min_tx)
715 : {
716 0 : bsp->args.detection_multiplier = detection_multiplier;
717 0 : bsp->args.min_rx = min_rx;
718 0 : bsp->args.min_tx = min_tx;
719 0 : }
720 :
721 0 : void bfd_sess_set_auto_source(struct bfd_session_params *bsp, bool enable)
722 : {
723 0 : if (bsp->auto_source == enable)
724 : return;
725 :
726 0 : bsp->auto_source = enable;
727 0 : if (enable)
728 0 : bfd_source_cache_get(bsp);
729 : else
730 0 : bfd_source_cache_put(bsp);
731 : }
732 :
733 0 : void bfd_sess_install(struct bfd_session_params *bsp)
734 : {
735 0 : bsp->lastev = BSE_INSTALL;
736 0 : event_add_event(bsglobal.tm, _bfd_sess_send, bsp, 0, &bsp->installev);
737 0 : }
738 :
739 0 : void bfd_sess_uninstall(struct bfd_session_params *bsp)
740 : {
741 0 : bsp->lastev = BSE_UNINSTALL;
742 0 : event_add_event(bsglobal.tm, _bfd_sess_send, bsp, 0, &bsp->installev);
743 0 : }
744 :
745 0 : enum bfd_session_state bfd_sess_status(const struct bfd_session_params *bsp)
746 : {
747 0 : return bsp->bss.state;
748 : }
749 :
750 0 : uint8_t bfd_sess_hop_count(const struct bfd_session_params *bsp)
751 : {
752 0 : return bsp->args.hops;
753 : }
754 :
755 0 : const char *bfd_sess_profile(const struct bfd_session_params *bsp)
756 : {
757 0 : return bsp->args.profilelen ? bsp->args.profile : NULL;
758 : }
759 :
760 0 : void bfd_sess_addresses(const struct bfd_session_params *bsp, int *family,
761 : struct in6_addr *src, struct in6_addr *dst)
762 : {
763 0 : *family = bsp->args.family;
764 0 : if (src)
765 0 : *src = bsp->args.src;
766 0 : if (dst)
767 0 : *dst = bsp->args.dst;
768 0 : }
769 :
770 0 : const char *bfd_sess_interface(const struct bfd_session_params *bsp)
771 : {
772 0 : if (bsp->args.ifnamelen)
773 0 : return bsp->args.ifname;
774 :
775 : return NULL;
776 : }
777 :
778 0 : const char *bfd_sess_vrf(const struct bfd_session_params *bsp)
779 : {
780 0 : return vrf_id_to_name(bsp->args.vrf_id);
781 : }
782 :
783 0 : vrf_id_t bfd_sess_vrf_id(const struct bfd_session_params *bsp)
784 : {
785 0 : return bsp->args.vrf_id;
786 : }
787 :
788 0 : bool bfd_sess_cbit(const struct bfd_session_params *bsp)
789 : {
790 0 : return bsp->args.cbit;
791 : }
792 :
793 0 : void bfd_sess_timers(const struct bfd_session_params *bsp,
794 : uint8_t *detection_multiplier, uint32_t *min_rx,
795 : uint32_t *min_tx)
796 : {
797 0 : *detection_multiplier = bsp->args.detection_multiplier;
798 0 : *min_rx = bsp->args.min_rx;
799 0 : *min_tx = bsp->args.min_tx;
800 0 : }
801 :
802 0 : bool bfd_sess_auto_source(const struct bfd_session_params *bsp)
803 : {
804 0 : return bsp->auto_source;
805 : }
806 :
807 0 : void bfd_sess_show(struct vty *vty, struct json_object *json,
808 : struct bfd_session_params *bsp)
809 : {
810 0 : json_object *json_bfd = NULL;
811 0 : char time_buf[64];
812 :
813 0 : if (!bsp)
814 0 : return;
815 :
816 : /* Show type. */
817 0 : if (json) {
818 0 : json_bfd = json_object_new_object();
819 0 : if (bsp->args.mhop)
820 0 : json_object_string_add(json_bfd, "type", "multi hop");
821 : else
822 0 : json_object_string_add(json_bfd, "type", "single hop");
823 : } else
824 0 : vty_out(vty, " BFD: Type: %s\n",
825 0 : bsp->args.mhop ? "multi hop" : "single hop");
826 :
827 : /* Show configuration. */
828 0 : if (json) {
829 0 : json_object_int_add(json_bfd, "detectMultiplier",
830 0 : bsp->args.detection_multiplier);
831 0 : json_object_int_add(json_bfd, "rxMinInterval",
832 0 : bsp->args.min_rx);
833 0 : json_object_int_add(json_bfd, "txMinInterval",
834 0 : bsp->args.min_tx);
835 : } else {
836 0 : vty_out(vty,
837 : " Detect Multiplier: %d, Min Rx interval: %d, Min Tx interval: %d\n",
838 : bsp->args.detection_multiplier, bsp->args.min_rx,
839 : bsp->args.min_tx);
840 : }
841 :
842 0 : bfd_last_update(bsp->bss.last_event, time_buf, sizeof(time_buf));
843 0 : if (json) {
844 0 : json_object_string_add(json_bfd, "status",
845 0 : bfd_get_status_str(bsp->bss.state));
846 0 : json_object_string_add(json_bfd, "lastUpdate", time_buf);
847 : } else
848 0 : vty_out(vty, " Status: %s, Last update: %s\n",
849 0 : bfd_get_status_str(bsp->bss.state), time_buf);
850 :
851 0 : if (json)
852 0 : json_object_object_add(json, "peerBfdInfo", json_bfd);
853 : else
854 0 : vty_out(vty, "\n");
855 : }
856 :
857 : /*
858 : * Zebra communication related.
859 : */
860 :
861 : /**
862 : * Callback for reinstallation of all registered BFD sessions.
863 : *
864 : * Use this as `zclient` `bfd_dest_replay` callback.
865 : */
866 0 : int zclient_bfd_session_replay(ZAPI_CALLBACK_ARGS)
867 : {
868 0 : struct bfd_session_params *bsp;
869 :
870 0 : if (!zclient->bfd_integration)
871 : return 0;
872 :
873 : /* Do nothing when shutting down. */
874 0 : if (bsglobal.shutting_down)
875 : return 0;
876 :
877 0 : if (bsglobal.debugging)
878 0 : zlog_debug("%s: sending all sessions registered", __func__);
879 :
880 : /* Send the client registration */
881 0 : bfd_client_sendmsg(zclient, ZEBRA_BFD_CLIENT_REGISTER, vrf_id);
882 :
883 : /* Replay all activated peers. */
884 0 : TAILQ_FOREACH (bsp, &bsglobal.bsplist, entry) {
885 : /* Skip not installed sessions. */
886 0 : if (!bsp->installed)
887 0 : continue;
888 :
889 : /* We are reconnecting, so we must send installation. */
890 0 : bsp->installed = false;
891 :
892 : /* Cancel any pending installation request. */
893 0 : EVENT_OFF(bsp->installev);
894 :
895 : /* Ask for installation. */
896 0 : bsp->lastev = BSE_INSTALL;
897 0 : event_execute(bsglobal.tm, _bfd_sess_send, bsp, 0, NULL);
898 : }
899 :
900 : return 0;
901 : }
902 :
903 0 : int zclient_bfd_session_update(ZAPI_CALLBACK_ARGS)
904 : {
905 0 : struct bfd_session_params *bsp, *bspn;
906 0 : size_t sessions_updated = 0;
907 0 : struct interface *ifp;
908 0 : int remote_cbit = false;
909 0 : int state = BFD_STATUS_UNKNOWN;
910 0 : time_t now;
911 0 : size_t addrlen;
912 0 : struct prefix dp;
913 0 : struct prefix sp;
914 0 : char ifstr[128], cbitstr[32];
915 :
916 0 : if (!zclient->bfd_integration)
917 : return 0;
918 :
919 : /* Do nothing when shutting down. */
920 0 : if (bsglobal.shutting_down)
921 : return 0;
922 :
923 0 : ifp = bfd_get_peer_info(zclient->ibuf, &dp, &sp, &state, &remote_cbit,
924 : vrf_id);
925 : /*
926 : * When interface lookup fails or an invalid stream is read, we must
927 : * not proceed otherwise it will trigger an assertion while checking
928 : * family type below.
929 : */
930 0 : if (dp.family == 0 || sp.family == 0)
931 : return 0;
932 :
933 0 : if (bsglobal.debugging) {
934 0 : ifstr[0] = 0;
935 0 : if (ifp)
936 0 : snprintf(ifstr, sizeof(ifstr), " (interface %s)",
937 0 : ifp->name);
938 :
939 0 : snprintf(cbitstr, sizeof(cbitstr), " (CPI bit %s)",
940 0 : remote_cbit ? "yes" : "no");
941 :
942 0 : zlog_debug("%s: %pFX -> %pFX%s VRF %s(%u)%s: %s", __func__, &sp,
943 : &dp, ifstr, vrf_id_to_name(vrf_id), vrf_id, cbitstr,
944 : bfd_get_status_str(state));
945 : }
946 :
947 0 : switch (dp.family) {
948 : case AF_INET:
949 : addrlen = sizeof(struct in_addr);
950 : break;
951 0 : case AF_INET6:
952 0 : addrlen = sizeof(struct in6_addr);
953 0 : break;
954 :
955 : default:
956 : /* Unexpected value. */
957 0 : assert(0);
958 : break;
959 : }
960 :
961 : /* Cache current time to avoid multiple monotime clock calls. */
962 0 : now = monotime(NULL);
963 :
964 : /* Notify all matching sessions about update. */
965 0 : TAILQ_FOREACH_SAFE (bsp, &bsglobal.bsplist, entry, bspn) {
966 : /* Skip not installed entries. */
967 0 : if (!bsp->installed)
968 0 : continue;
969 : /* Skip different VRFs. */
970 0 : if (bsp->args.vrf_id != vrf_id)
971 0 : continue;
972 : /* Skip different families. */
973 0 : if (bsp->args.family != dp.family)
974 0 : continue;
975 : /* Skip different interface. */
976 0 : if (bsp->args.ifnamelen && ifp
977 0 : && strcmp(bsp->args.ifname, ifp->name) != 0)
978 0 : continue;
979 : /* Skip non matching destination addresses. */
980 0 : if (memcmp(&bsp->args.dst, &dp.u, addrlen) != 0)
981 0 : continue;
982 : /*
983 : * Source comparison test:
984 : * We will only compare source if BFD daemon provided the
985 : * source address and the protocol set a source address in
986 : * the configuration otherwise we'll just skip it.
987 : */
988 0 : if (sp.family && memcmp(&bsp->args.src, &i6a_zero, addrlen) != 0
989 0 : && memcmp(&sp.u, &i6a_zero, addrlen) != 0
990 0 : && memcmp(&bsp->args.src, &sp.u, addrlen) != 0)
991 0 : continue;
992 : /* No session state change. */
993 0 : if ((int)bsp->bss.state == state)
994 0 : continue;
995 :
996 0 : bsp->bss.last_event = now;
997 0 : bsp->bss.previous_state = bsp->bss.state;
998 0 : bsp->bss.state = state;
999 0 : bsp->bss.remote_cbit = remote_cbit;
1000 0 : bsp->updatecb(bsp, &bsp->bss, bsp->arg);
1001 0 : sessions_updated++;
1002 : }
1003 :
1004 0 : if (bsglobal.debugging)
1005 0 : zlog_debug("%s: sessions updated: %zu", __func__,
1006 : sessions_updated);
1007 :
1008 : return 0;
1009 : }
1010 :
1011 : /**
1012 : * Frees all allocated resources and stops any activity.
1013 : *
1014 : * Must be called after every BFD session has been successfully
1015 : * unconfigured otherwise this function will `free()` any available
1016 : * session causing existing pointers to dangle.
1017 : *
1018 : * This is just a comment, in practice it will be called by the FRR
1019 : * library late finish hook. \see `bfd_protocol_integration_init`.
1020 : */
1021 2 : static int bfd_protocol_integration_finish(void)
1022 : {
1023 2 : if (bsglobal.zc == NULL)
1024 : return 0;
1025 :
1026 2 : while (!TAILQ_EMPTY(&bsglobal.bsplist)) {
1027 0 : struct bfd_session_params *session =
1028 : TAILQ_FIRST(&bsglobal.bsplist);
1029 0 : bfd_sess_free(&session);
1030 : }
1031 :
1032 : /*
1033 : * BFD source cache is linked to sessions, if all sessions are gone
1034 : * then the source cache must be empty.
1035 : */
1036 2 : if (!SLIST_EMPTY(&bsglobal.source_list))
1037 0 : zlog_warn("BFD integration source cache not empty");
1038 :
1039 : return 0;
1040 : }
1041 :
1042 2 : void bfd_protocol_integration_init(struct zclient *zc, struct event_loop *tm)
1043 : {
1044 : /* Initialize data structure. */
1045 2 : TAILQ_INIT(&bsglobal.bsplist);
1046 2 : SLIST_INIT(&bsglobal.source_list);
1047 :
1048 : /* Copy pointers. */
1049 2 : bsglobal.zc = zc;
1050 2 : bsglobal.tm = tm;
1051 :
1052 : /* Enable BFD callbacks. */
1053 2 : zc->bfd_integration = true;
1054 :
1055 : /* Send the client registration */
1056 2 : bfd_client_sendmsg(zc, ZEBRA_BFD_CLIENT_REGISTER, VRF_DEFAULT);
1057 :
1058 2 : hook_register(frr_fini, bfd_protocol_integration_finish);
1059 2 : }
1060 :
1061 0 : void bfd_protocol_integration_set_debug(bool enable)
1062 : {
1063 0 : bsglobal.debugging = enable;
1064 0 : }
1065 :
1066 2 : void bfd_protocol_integration_set_shutdown(bool enable)
1067 : {
1068 2 : bsglobal.shutting_down = enable;
1069 2 : }
1070 :
1071 0 : bool bfd_protocol_integration_debug(void)
1072 : {
1073 0 : return bsglobal.debugging;
1074 : }
1075 :
1076 0 : bool bfd_protocol_integration_shutting_down(void)
1077 : {
1078 0 : return bsglobal.shutting_down;
1079 : }
1080 :
1081 : /*
1082 : * BFD automatic source selection
1083 : *
1084 : * This feature will use the next hop tracking (NHT) provided by zebra
1085 : * to find out the source address by looking at the output interface.
1086 : *
1087 : * When the interface address / routing table change we'll be notified
1088 : * and be able to update the source address accordingly.
1089 : *
1090 : * <daemon> zebra
1091 : * |
1092 : * +-----------------+
1093 : * | BFD session set |
1094 : * | to auto source |
1095 : * +-----------------+
1096 : * |
1097 : * \ +-----------------+
1098 : * --------------> | Resolves |
1099 : * | destination |
1100 : * | address |
1101 : * +-----------------+
1102 : * |
1103 : * +-----------------+ /
1104 : * | Sets resolved | <----------
1105 : * | source address |
1106 : * +-----------------+
1107 : */
1108 : static bool
1109 0 : bfd_source_cache_session_match(const struct bfd_source_cache *source,
1110 : const struct bfd_session_params *session)
1111 : {
1112 0 : const struct in_addr *address;
1113 0 : const struct in6_addr *address_v6;
1114 :
1115 0 : if (session->args.vrf_id != source->vrf_id)
1116 : return false;
1117 0 : if (session->args.family != source->address.family)
1118 : return false;
1119 :
1120 0 : switch (session->args.family) {
1121 0 : case AF_INET:
1122 0 : address = (const struct in_addr *)&session->args.dst;
1123 0 : if (address->s_addr != source->address.u.prefix4.s_addr)
1124 0 : return false;
1125 : break;
1126 0 : case AF_INET6:
1127 0 : address_v6 = &session->args.dst;
1128 0 : if (memcmp(address_v6, &source->address.u.prefix6,
1129 : sizeof(struct in6_addr)))
1130 0 : return false;
1131 : break;
1132 : default:
1133 : return false;
1134 : }
1135 :
1136 : return true;
1137 : }
1138 :
1139 : static struct bfd_source_cache *
1140 0 : bfd_source_cache_find(vrf_id_t vrf_id, const struct prefix *prefix)
1141 : {
1142 0 : struct bfd_source_cache *source;
1143 :
1144 0 : SLIST_FOREACH (source, &bsglobal.source_list, entry) {
1145 0 : if (source->vrf_id != vrf_id)
1146 0 : continue;
1147 0 : if (!prefix_same(&source->address, prefix))
1148 0 : continue;
1149 :
1150 : return source;
1151 : }
1152 :
1153 : return NULL;
1154 : }
1155 :
1156 0 : static void bfd_source_cache_get(struct bfd_session_params *session)
1157 : {
1158 0 : struct bfd_source_cache *source;
1159 0 : struct prefix target = {};
1160 :
1161 0 : switch (session->args.family) {
1162 0 : case AF_INET:
1163 0 : target.family = AF_INET;
1164 0 : target.prefixlen = IPV4_MAX_BITLEN;
1165 0 : memcpy(&target.u.prefix4, &session->args.dst,
1166 : sizeof(struct in_addr));
1167 0 : break;
1168 0 : case AF_INET6:
1169 0 : target.family = AF_INET6;
1170 0 : target.prefixlen = IPV6_MAX_BITLEN;
1171 0 : memcpy(&target.u.prefix6, &session->args.dst,
1172 : sizeof(struct in6_addr));
1173 0 : break;
1174 : default:
1175 : return;
1176 : }
1177 :
1178 0 : source = bfd_source_cache_find(session->args.vrf_id, &target);
1179 0 : if (source) {
1180 0 : if (session->source_cache == source)
1181 : return;
1182 :
1183 0 : bfd_source_cache_put(session);
1184 0 : session->source_cache = source;
1185 0 : source->refcount++;
1186 0 : return;
1187 : }
1188 :
1189 0 : source = XCALLOC(MTYPE_BFD_SOURCE, sizeof(*source));
1190 0 : prefix_copy(&source->address, &target);
1191 0 : source->vrf_id = session->args.vrf_id;
1192 0 : SLIST_INSERT_HEAD(&bsglobal.source_list, source, entry);
1193 :
1194 0 : bfd_source_cache_put(session);
1195 0 : session->source_cache = source;
1196 0 : source->refcount = 1;
1197 :
1198 0 : return;
1199 : }
1200 :
1201 0 : static void bfd_source_cache_put(struct bfd_session_params *session)
1202 : {
1203 0 : if (session->source_cache == NULL)
1204 : return;
1205 :
1206 0 : session->source_cache->refcount--;
1207 0 : if (session->source_cache->refcount > 0) {
1208 0 : session->source_cache = NULL;
1209 0 : return;
1210 : }
1211 :
1212 0 : SLIST_REMOVE(&bsglobal.source_list, session->source_cache,
1213 : bfd_source_cache, entry);
1214 0 : XFREE(MTYPE_BFD_SOURCE, session->source_cache);
1215 : }
1216 :
1217 : /** Updates BFD running session if source address has changed. */
1218 : static void
1219 0 : bfd_source_cache_update_session(const struct bfd_source_cache *source,
1220 : struct bfd_session_params *session)
1221 : {
1222 0 : const struct in_addr *address;
1223 0 : const struct in6_addr *address_v6;
1224 :
1225 0 : switch (session->args.family) {
1226 0 : case AF_INET:
1227 0 : address = (const struct in_addr *)&session->args.src;
1228 0 : if (memcmp(address, &source->source.u.prefix4,
1229 : sizeof(struct in_addr)) == 0)
1230 : return;
1231 :
1232 0 : _bfd_sess_remove(session);
1233 0 : memcpy(&session->args.src, &source->source.u.prefix4,
1234 : sizeof(struct in_addr));
1235 0 : break;
1236 0 : case AF_INET6:
1237 0 : address_v6 = &session->args.src;
1238 0 : if (memcmp(address_v6, &source->source.u.prefix6,
1239 : sizeof(struct in6_addr)) == 0)
1240 : return;
1241 :
1242 0 : _bfd_sess_remove(session);
1243 0 : memcpy(&session->args.src, &source->source.u.prefix6,
1244 : sizeof(struct in6_addr));
1245 0 : break;
1246 : default:
1247 : return;
1248 : }
1249 :
1250 0 : bfd_sess_install(session);
1251 : }
1252 :
1253 : static void
1254 0 : bfd_source_cache_update_sessions(const struct bfd_source_cache *source)
1255 : {
1256 0 : struct bfd_session_params *session;
1257 :
1258 0 : if (!source->valid)
1259 : return;
1260 :
1261 0 : TAILQ_FOREACH (session, &bsglobal.bsplist, entry) {
1262 0 : if (!session->auto_source)
1263 0 : continue;
1264 0 : if (!bfd_source_cache_session_match(source, session))
1265 0 : continue;
1266 :
1267 0 : bfd_source_cache_update_session(source, session);
1268 : }
1269 : }
1270 :
1271 : /**
1272 : * Try to translate next hop information into source address.
1273 : *
1274 : * \returns `true` if source changed otherwise `false`.
1275 : */
1276 0 : static bool bfd_source_cache_update(struct bfd_source_cache *source,
1277 : const struct zapi_route *route)
1278 : {
1279 0 : size_t nh_index;
1280 :
1281 0 : for (nh_index = 0; nh_index < route->nexthop_num; nh_index++) {
1282 0 : const struct zapi_nexthop *nh = &route->nexthops[nh_index];
1283 0 : const struct interface *interface;
1284 0 : const struct connected *connected;
1285 0 : const struct listnode *node;
1286 :
1287 0 : interface = if_lookup_by_index(nh->ifindex, nh->vrf_id);
1288 0 : if (interface == NULL) {
1289 0 : zlog_err("next hop interface not found (index %d)",
1290 : nh->ifindex);
1291 0 : continue;
1292 : }
1293 :
1294 0 : for (ALL_LIST_ELEMENTS_RO(interface->connected, node,
1295 : connected)) {
1296 0 : if (source->address.family !=
1297 0 : connected->address->family)
1298 0 : continue;
1299 0 : if (prefix_same(connected->address, &source->source))
1300 : return false;
1301 : /*
1302 : * Skip link-local as it is only useful for single hop
1303 : * and in that case no source is specified usually.
1304 : */
1305 0 : if (source->address.family == AF_INET6 &&
1306 0 : IN6_IS_ADDR_LINKLOCAL(
1307 : &connected->address->u.prefix6))
1308 0 : continue;
1309 :
1310 0 : prefix_copy(&source->source, connected->address);
1311 0 : source->valid = true;
1312 0 : return true;
1313 : }
1314 : }
1315 :
1316 0 : memset(&source->source, 0, sizeof(source->source));
1317 0 : source->valid = false;
1318 0 : return false;
1319 : }
1320 :
1321 0 : int bfd_nht_update(const struct prefix *match, const struct zapi_route *route)
1322 : {
1323 0 : struct bfd_source_cache *source;
1324 :
1325 0 : if (bsglobal.debugging)
1326 0 : zlog_debug("BFD NHT update for %pFX", &route->prefix);
1327 :
1328 0 : SLIST_FOREACH (source, &bsglobal.source_list, entry) {
1329 0 : if (source->vrf_id != route->vrf_id)
1330 0 : continue;
1331 0 : if (!prefix_same(match, &source->address))
1332 0 : continue;
1333 0 : if (bfd_source_cache_update(source, route))
1334 0 : bfd_source_cache_update_sessions(source);
1335 : }
1336 :
1337 0 : return 0;
1338 : }
|
