break;"""
def fragment_to_rule(ipproto, rules):
- ast = parse_ast(rules, parse_frag_expr, True)
+ ast = parse_ast(rules, parse_frag_expr, False)
return "if (!( " + ast.write(ipproto) + " )) break;"
def len_to_rule(rules):
def port_to_rule(ty, rules):
if ty == "port" :
ast = parse_ast(rules, parse_numbers_expr, True)
- return "if (!ports_valid) break;\nif (!( " + ast.write("sport", "dport") + " )) break;"
+ return "if (sport == -1 || dport == -1) break;\nif (!( " + ast.write("sport", "dport") + " )) break;"
ast = parse_ast(rules, parse_numbers_expr, True)
- return "if (!ports_valid) break;\nif (!( " + ast.write(ty) + " )) break;"
+ return "if (" + ty + " == -1) break;\nif (!( " + ast.write(ty) + " )) break;"
def tcp_flags_to_rule(rules):
ast = parse_ast(rules, parse_bit_expr, False)
rules6 = ""
rules4 = ""
use_v6_frags = False
- rulecnt = 0
+ stats_rulecnt = 0
ratelimitcnt = 0
v4persrcratelimits = []
+ v5persrcratelimits = []
v6persrcratelimits = []
lastrule = None
else:
continue
- def write_rule(r):
- global rules4, rules6
- if proto == 6:
+ # LLVM can be pretty bad at optimizing out common subexpressions. Ideally we'd optimize
+ # by pulling common subexpressions in back-to-back rules out into a single check, but
+ # that's a bunch of work that LLVM really should do for us. Instead, we blindly guess
+ # that source-address is the least likely to be a common subexpression and rely on LLVM
+ # managing to pull out common subexpressions as long as they're the first check(s). By
+ # placing source-address checks last, LLVM should do at least some work for us.
+ # See https://bugs.llvm.org/show_bug.cgi?id=52455
+ last_checks = ""
+ def write_rule(r, place_at_end=False):
+ global rules4, rules6, last_checks
+ if place_at_end:
+ last_checks += "\t\t" + r.replace("\n", " \\\n\t\t") + " \\\n"
+ elif proto == 6:
rules6 += "\t\t" + r.replace("\n", " \\\n\t\t") + " \\\n"
else:
rules4 += "\t\t" + r.replace("\n", " \\\n\t\t") + " \\\n"
else:
offset = None
if step.strip().startswith("src"):
- write_rule(ip_to_rule(proto, nets[0], "saddr", offset))
+ write_rule(ip_to_rule(proto, nets[0], "saddr", offset), True)
else:
write_rule(ip_to_rule(proto, nets[0], "daddr", offset))
elif step.strip().startswith("proto") and proto == 4:
else:
assert False
+ if proto == 6:
+ rules6 += last_checks
+ else:
+ rules4 += last_checks
+
# Now write the match handling!
first_action = None
- stats_action = None
+ stats_action = ""
last_action = None
for community in line.split("("):
if not community.startswith("generic, "):
elif exp == 0xff:
# NaN/INF. Just treat as INF and accept
first_action = None
- elif exp <= 127: # < 1
+ elif exp < 127: # < 1
first_action = "{stats_replace}\nreturn XDP_DROP;"
- elif exp >= 127 + 63: # The count won't even fit in 64-bits, just accept
+ elif exp >= 127 + 29: # We can't handle the precision required with ns this high
first_action = None
else:
mantissa = low_bytes & ((1 << 23) - 1)
value = 1.0 + mantissa / (2**23)
value *= 2**(exp-127)
+
+ first_action = "int64_t time_masked = bpf_ktime_get_ns() & RATE_TIME_MASK;\n"
+ first_action += f"int64_t per_pkt_ns = (1000000000LL << RATE_BUCKET_INTEGER_BITS) / {math.floor(value)};\n"
if ty == "0x8006" or ty == "0x8306":
- accessor = "rate->rate.sent_bytes"
+ first_action += "uint64_t amt = data_end - pktdata;\n"
else:
- accessor = "rate->rate.sent_packets"
- # Note that int64_t will overflow after 292 years of uptime
- first_action = "int64_t time = bpf_ktime_get_ns();\n"
- first_action += "uint64_t allowed_since_last = 0;\n"
+ first_action += "uint64_t amt = 1;\n"
if ty == "0x8006" or ty == "0x800c":
- spin_lock = "bpf_spin_lock(&rate->lock);"
- spin_unlock = "bpf_spin_unlock(&rate->lock);"
first_action += f"const uint32_t ratelimitidx = {ratelimitcnt};\n"
first_action += "struct ratelimit *rate = bpf_map_lookup_elem(&rate_map, &ratelimitidx);\n"
ratelimitcnt += 1
+ first_action += "int matched = 0;\n"
+ first_action += "DO_RATE_LIMIT(bpf_spin_lock(&rate->lock), rate, time_masked, amt, per_pkt_ns, matched);\n"
+ first_action += "if (rate) { bpf_spin_unlock(&rate->lock); }\n"
else:
- spin_lock = "/* No locking as we're per-CPU */"
- spin_unlock = "/* No locking as we're per-CPU */"
if proto == 4:
if mid_byte > 32:
continue
first_action += f"const uint32_t srcip = ip->saddr & MASK4({mid_byte});\n"
first_action += f"void *rate_map = &v4_src_rate_{len(v4persrcratelimits)};\n"
- v4persrcratelimits.append((high_byte + 1) * 1024)
+ first_action += f"int matched = check_v4_persrc_ratelimit(srcip, rate_map, {(high_byte + 1) * 4096}, time_masked, amt, per_pkt_ns);\n"
+ v4persrcratelimits.append((high_byte + 1) * 4096)
+ elif mid_byte <= 64:
+ first_action += f"const uint64_t srcip = BE128BEHIGH64(ip6->saddr & MASK6({mid_byte}));\n"
+ first_action += f"void *rate_map = &v5_src_rate_{len(v5persrcratelimits)};\n"
+ first_action += f"int matched = check_v5_persrc_ratelimit(srcip, rate_map, {(high_byte + 1) * 4096}, time_masked, amt, per_pkt_ns);\n"
+ v5persrcratelimits.append((high_byte + 1) * 4096)
else:
if mid_byte > 128:
continue
first_action += f"const uint128_t srcip = ip6->saddr & MASK6({mid_byte});\n"
first_action += f"void *rate_map = &v6_src_rate_{len(v6persrcratelimits)};\n"
- v6persrcratelimits.append((high_byte + 1) * 1024)
- first_action += f"struct percpu_ratelimit *rate = bpf_map_lookup_elem(rate_map, &srcip);\n"
- first_action += "if (rate) {\n"
- first_action += f"\t{spin_lock}\n"
- first_action += f"\tif (likely({accessor} > 0))" + " {\n"
- first_action += "\t\tint64_t diff = time - rate->sent_time;\n"
- # Unlikely or not, if the flow is slow, take a perf hit (though with the else if branch it doesn't matter)
- first_action += "\t\tif (unlikely(diff > 1000000000))\n"
- first_action += f"\t\t\t{accessor} = 0;\n"
- first_action += "\t\telse if (likely(diff > 0))\n"
- first_action += f"\t\t\tallowed_since_last = ((uint64_t)diff) * {math.floor(value)} / 1000000000;\n"
- first_action += "\t}\n"
- first_action += f"\tif ({accessor} - ((int64_t)allowed_since_last) <= 0)" + " {\n"
- if ty == "0x8006" or ty == "0x8306":
- first_action += f"\t\t{accessor} = data_end - pktdata;\n"
- else:
- first_action += f"\t\t{accessor} = 1;\n"
- first_action += "\t\trate->sent_time = time;\n"
- first_action += f"\t\t{spin_unlock}\n"
- first_action += "\t} else {\n"
- first_action += f"\t\t{spin_unlock}\n"
- first_action += "\t\t{stats_replace}\n"
- first_action += "\t\treturn XDP_DROP;\n"
- first_action += "\t}\n"
- if ty == "0x8306" or ty == "0x830c":
- first_action += "} else {\n"
- first_action += "\tstruct percpu_ratelimit new_rate = { .sent_time = time, };\n"
- first_action += "\trate = &new_rate;\n"
- if ty == "0x8006" or ty == "0x8306":
- first_action += f"\t\t{accessor} = data_end - pktdata;\n"
- else:
- first_action += f"\t\t{accessor} = 1;\n"
- first_action += "\tbpf_map_update_elem(rate_map, &srcip, rate, BPF_ANY);\n"
+ first_action += f"int matched = check_v6_persrc_ratelimit(srcip, rate_map, {(high_byte + 1) * 4096}, time_masked, amt, per_pkt_ns);\n"
+ v6persrcratelimits.append((high_byte + 1) * 4096)
+ first_action += "if (matched) {\n"
+ first_action += "\t{stats_replace}\n"
+ first_action += "\treturn XDP_DROP;\n"
first_action += "}\n"
elif ty == "0x8007":
if low_bytes & 1 == 0:
last_action = "return XDP_PASS;"
if low_bytes & 2 == 2:
- stats_action = f"const uint32_t ruleidx = STATIC_RULE_CNT + {rulecnt};\n"
+ stats_action = f"const uint32_t ruleidx = STATIC_RULE_CNT + {stats_rulecnt};\n"
stats_action += "INCREMENT_MATCH(ruleidx);"
elif ty == "0x8008":
assert False # We do not implement the redirect action
write_rule("ip6->flow_lbl[0] = (ip6->flow_lbl[0] & 0x3f) | " + str((low_bytes & 3) << 6) + ";")
if first_action is not None:
write_rule(first_action.replace("{stats_replace}", stats_action))
- if stats_action is not None and (first_action is None or "{stats_replace}" not in first_action):
+ if stats_action != "" and (first_action is None or "{stats_replace}" not in first_action):
write_rule(stats_action)
if last_action is not None:
write_rule(last_action)
rules6 += "\t} while(0);\\\n"
else:
rules4 += "\t} while(0);\\\n"
- rulecnt += 1
+ if stats_action != "":
+ print(rule)
+ stats_rulecnt += 1
lastrule = None
out.write("\n")
- out.write(f"#define RULECNT {rulecnt}\n")
+ out.write(f"#define STATS_RULECNT {stats_rulecnt}\n")
if ratelimitcnt != 0:
out.write(f"#define RATE_CNT {ratelimitcnt}\n")
if rules4 != "":
out.write("#define PARSE_V6_FRAG PARSE\n")
with open("maps.h", "w") as out:
for idx, limit in enumerate(v4persrcratelimits):
- out.write(f"V4_SRC_RATE_DEFINE({idx}, {limit})\n")
+ out.write(f"SRC_RATE_DEFINE(4, {idx}, {limit})\n")
+ for idx, limit in enumerate(v5persrcratelimits):
+ out.write(f"SRC_RATE_DEFINE(5, {idx}, {limit})\n")
for idx, limit in enumerate(v6persrcratelimits):
- out.write(f"V6_SRC_RATE_DEFINE({idx}, {limit})\n")
+ out.write(f"SRC_RATE_DEFINE(6, {idx}, {limit})\n")