Skip to content

Commit

Permalink
Merge pull request #9 from stackhpc/upstream/2024.1-2024-11-04
Browse files Browse the repository at this point in the history 
Synchronise 2024.1 with upstream
  • Loading branch information
@priteau
priteau authored Nov 4, 2024
2 parents cc88788 + c0ca1a5 commit 684a8c8
Show file tree
Hide file tree
Showing 14 changed files with 152 additions and 185 deletions.
62 changes: 37 additions & 25 deletions elements/amphora-agent/static/usr/local/bin/lvs-masquerade.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ set -e

usage() {
echo
echo "Usage: $(basename "$0") [add|delete] [ipv4|ipv6] <interface>"
echo "Usage: $(basename "$0") [add|delete] [ipv4|ipv6] <interface> [sriov]"
echo
exit 1
}
Expand All @@ -38,21 +38,25 @@ if [ "$1" == "add" ]; then
nft add rule ip octavia-ipv4 ip-udp-masq oifname "$3" meta l4proto udp masquerade
nft add chain ip octavia-ipv4 ip-sctp-masq { type nat hook postrouting priority 100\;}
nft add rule ip octavia-ipv4 ip-sctp-masq oifname "$3" meta l4proto sctp masquerade
nft -- add chain ip octavia-ipv4 prerouting { type filter hook prerouting priority -300 \; }
nft add rule ip octavia-ipv4 prerouting iifname "$3" meta l4proto tcp notrack
nft -- add chain ip octavia-ipv4 output { type filter hook output priority -300 \; }
nft add rule ip octavia-ipv4 output oifname "$3" meta l4proto tcp notrack
if ! [ "$4" == "sriov" ]; then
nft -- add chain ip octavia-ipv4 prerouting { type filter hook prerouting priority -300 \; }
nft add rule ip octavia-ipv4 prerouting iifname "$3" meta l4proto tcp notrack
nft -- add chain ip octavia-ipv4 output { type filter hook output priority -300 \; }
nft add rule ip octavia-ipv4 output oifname "$3" meta l4proto tcp notrack
fi

elif [ "$2" == "ipv6" ]; then
nft add table ip6 octavia-ipv6
nft add chain ip6 octavia-ipv6 ip6-udp-masq { type nat hook postrouting priority 100\;}
nft add rule ip6 octavia-ipv6 ip6-udp-masq oifname "$3" meta l4proto udp masquerade
nft add chain ip6 octavia-ipv6 ip6-sctp-masq { type nat hook postrouting priority 100\;}
nft add rule ip6 octavia-ipv6 ip6-sctp-masq oifname "$3" meta l4proto sctp masquerade
nft -- add chain ip6 octavia-ipv6 prerouting { type filter hook prerouting priority -300 \; }
nft add rule ip6 octavia-ipv6 prerouting iifname "$3" meta l4proto tcp notrack
nft -- add chain ip6 octavia-ipv6 output { type filter hook output priority -300 \; }
nft add rule ip6 octavia-ipv6 output oifname "$3" meta l4proto tcp notrack
if ! [ "$4" == "sriov" ]; then
nft -- add chain ip6 octavia-ipv6 prerouting { type filter hook prerouting priority -300 \; }
nft add rule ip6 octavia-ipv6 prerouting iifname "$3" meta l4proto tcp notrack
nft -- add chain ip6 octavia-ipv6 output { type filter hook output priority -300 \; }
nft add rule ip6 octavia-ipv6 output oifname "$3" meta l4proto tcp notrack
fi
else
usage
fi
Expand All @@ -62,14 +66,18 @@ if [ "$1" == "add" ]; then
/sbin/iptables -t nat -A POSTROUTING -p udp -o $3 -j MASQUERADE
/sbin/iptables -t nat -A POSTROUTING -p sctp -o $3 -j MASQUERADE

/sbin/iptables -t raw -A PREROUTING -p tcp -i $3 -j NOTRACK
/sbin/iptables -t raw -A OUTPUT -p tcp -o $3 -j NOTRACK
if ! [ "$4" == "sriov" ]; then
/sbin/iptables -t raw -A PREROUTING -p tcp -i $3 -j NOTRACK
/sbin/iptables -t raw -A OUTPUT -p tcp -o $3 -j NOTRACK
fi
elif [ "$2" == "ipv6" ]; then
/sbin/ip6tables -t nat -A POSTROUTING -p udp -o $3 -j MASQUERADE
/sbin/ip6tables -t nat -A POSTROUTING -p sctp -o $3 -j MASQUERADE

/sbin/ip6tables -t raw -A PREROUTING -p tcp -i $3 -j NOTRACK
/sbin/ip6tables -t raw -A OUTPUT -p tcp -o $3 -j NOTRACK
if ! [ "$4" == "sriov" ]; then
/sbin/ip6tables -t raw -A PREROUTING -p tcp -i $3 -j NOTRACK
/sbin/ip6tables -t raw -A OUTPUT -p tcp -o $3 -j NOTRACK
fi
else
usage
fi
Expand All @@ -83,19 +91,21 @@ elif [ "$1" == "delete" ]; then
nft delete chain ip octavia-ipv4 ip-udp-masq
nft flush chain ip octavia-ipv4 ip-sctp-masq
nft delete chain ip octavia-ipv4 ip-sctp-masq
nft flush chain ip octavia-ipv4 prerouting
nft delete chain ip octavia-ipv4 prerouting
nft flush chain ip octavia-ipv4 output
nft delete chain ip octavia-ipv4 output
# Don't abort the script if these chains don't exist
nft flush chain ip octavia-ipv4 prerouting || true
nft delete chain ip octavia-ipv4 prerouting || true
nft flush chain ip octavia-ipv4 output || true
nft delete chain ip octavia-ipv4 output || true
elif [ "$2" == "ipv6" ]; then
nft flush chain ip6 octavia-ipv6 ip6-udp-masq
nft delete chain ip6 octavia-ipv6 ip6-udp-masq
nft flush chain ip6 octavia-ipv6 ip6-sctp-masq
nft delete chain ip6 octavia-ipv6 ip6-sctp-masq
nft flush chain ip6 octavia-ipv6 prerouting
nft delete chain ip6 octavia-ipv6 prerouting
nft flush chain ip6 octavia-ipv6 output
nft delete chain ip6 octavia-ipv6 output
# Don't abort the script if these chains don't exist
nft flush chain ip6 octavia-ipv6 prerouting || true
nft delete chain ip6 octavia-ipv6 prerouting || true
nft flush chain ip6 octavia-ipv6 output || true
nft delete chain ip6 octavia-ipv6 output || true
else
usage
fi
Expand All @@ -104,13 +114,15 @@ elif [ "$1" == "delete" ]; then
if [ "$2" == "ipv4" ]; then
/sbin/iptables -t nat -D POSTROUTING -p udp -o $3 -j MASQUERADE
/sbin/iptables -t nat -D POSTROUTING -p sctp -o $3 -j MASQUERADE
/sbin/iptables -t raw -D PREROUTING -p tcp -i $3 -j NOTRACK
/sbin/iptables -t raw -D OUTPUT -p tcp -o $3 -j NOTRACK
# Don't abort the script if these chains don't exist
/sbin/iptables -t raw -D PREROUTING -p tcp -i $3 -j NOTRACK || true
/sbin/iptables -t raw -D OUTPUT -p tcp -o $3 -j NOTRACK || true
elif [ "$2" == "ipv6" ]; then
/sbin/ip6tables -t nat -D POSTROUTING -p udp -o $3 -j MASQUERADE
/sbin/ip6tables -t nat -D POSTROUTING -p sctp -o $3 -j MASQUERADE
/sbin/ip6tables -t raw -D PREROUTING -p tcp -i $3 -j NOTRACK
/sbin/ip6tables -t raw -D OUTPUT -p tcp -o $3 -j NOTRACK
# Don't abort the script if these chains don't exist
/sbin/ip6tables -t raw -D PREROUTING -p tcp -i $3 -j NOTRACK || true
/sbin/ip6tables -t raw -D OUTPUT -p tcp -o $3 -j NOTRACK || true
else
usage
fi
Expand Down
6 changes: 4 additions & 2 deletions octavia/amphorae/backends/agent/api_server/osutils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,11 +75,13 @@ def write_vip_interface_file(self, interface, vips, mtu, vrrp_info,
is_sriov=is_sriov)
vip_interface.write()

def write_port_interface_file(self, interface, fixed_ips, mtu):
def write_port_interface_file(self, interface, fixed_ips, mtu,
is_sriov=False):
port_interface = interface_file.PortInterfaceFile(
name=interface,
mtu=mtu,
fixed_ips=fixed_ips)
fixed_ips=fixed_ips,
is_sriov=is_sriov)
port_interface.write()

@classmethod
Expand Down
9 changes: 5 additions & 4 deletions octavia/amphorae/backends/agent/api_server/plug.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ def _check_ip_addresses(self, fixed_ips):
socket.inet_pton(socket.AF_INET6, ip.get('ip_address'))

def plug_network(self, mac_address, fixed_ips, mtu=None,
vip_net_info=None):
vip_net_info=None, is_sriov=False):
try:
self._check_ip_addresses(fixed_ips=fixed_ips)
except socket.error:
Expand Down Expand Up @@ -189,15 +189,16 @@ def plug_network(self, mac_address, fixed_ips, mtu=None,
vips=rendered_vips,
mtu=mtu,
vrrp_info=vrrp_info,
fixed_ips=fixed_ips)
fixed_ips=fixed_ips,
is_sriov=is_sriov)
self._osutils.bring_interface_up(existing_interface, 'vip')
# Otherwise, we are just plugging a run-of-the-mill network
else:
# Write an updated config
self._osutils.write_port_interface_file(
interface=existing_interface,
fixed_ips=fixed_ips,
mtu=mtu)
mtu=mtu, is_sriov=is_sriov)
self._osutils.bring_interface_up(existing_interface, 'network')

util.send_member_advertisements(fixed_ips)
Expand All @@ -222,7 +223,7 @@ def plug_network(self, mac_address, fixed_ips, mtu=None,
self._osutils.write_port_interface_file(
interface=netns_interface,
fixed_ips=fixed_ips,
mtu=mtu)
mtu=mtu, is_sriov=is_sriov)

# Update the list of interfaces to add to the namespace
self._update_plugged_interfaces_file(netns_interface, mac_address)
Expand Down
5 changes: 3 additions & 2 deletions octavia/amphorae/backends/agent/api_server/server.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,8 @@ def plug_network(self):
return self._plug.plug_network(port_info['mac_address'],
port_info.get('fixed_ips'),
port_info.get('mtu'),
port_info.get('vip_net_info'))
port_info.get('vip_net_info'),
port_info.get('is_sriov'))

def upload_cert(self):
return certificate_update.upload_server_cert()
Expand Down Expand Up @@ -278,7 +279,7 @@ def set_interface_rules(self, ip_addr):
raise exceptions.BadRequest(
description='Invalid rules information') from e

nftable_utils.write_nftable_vip_rules_file(interface, rules_info)
nftable_utils.write_nftable_rules_file(interface, rules_info)

nftable_utils.load_nftables_file()

Expand Down
39 changes: 2 additions & 37 deletions octavia/amphorae/backends/utils/interface.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,47 +176,12 @@ def _normalize_ip_network(self, address):
ip_network = ipaddress.ip_network(address, strict=False)
return ip_network.compressed

def _setup_nftables_chain(self, interface):
# TODO(johnsom) Move this to pyroute2 when the nftables library
# improves.

# Create the nftable
cmd = [consts.NFT_CMD, consts.NFT_ADD, 'table', consts.NFT_FAMILY,
consts.NFT_VIP_TABLE]
try:
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
except Exception as e:
if hasattr(e, 'output'):
LOG.error(e.output)
else:
LOG.error(e)
raise

# Create the chain with -310 priority to put it in front of the
# lvs-masquerade configured chain
cmd = [consts.NFT_CMD, consts.NFT_ADD, 'chain', consts.NFT_FAMILY,
consts.NFT_VIP_TABLE, consts.NFT_VIP_CHAIN,
'{', 'type', 'filter', 'hook', 'ingress', 'device',
interface.name, 'priority', consts.NFT_SRIOV_PRIORITY, ';',
'policy', 'drop', ';', '}']
try:
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
except Exception as e:
if hasattr(e, 'output'):
LOG.error(e.output)
else:
LOG.error(e)
raise

nftable_utils.write_nftable_vip_rules_file(interface.name, [])

nftable_utils.load_nftables_file()

def up(self, interface):
LOG.info("Setting interface %s up", interface.name)

if interface.is_sriov:
self._setup_nftables_chain(interface)
nftable_utils.write_nftable_rules_file(interface.name, [])
nftable_utils.load_nftables_file()

with pyroute2.IPRoute() as ipr:
idx = ipr.link_lookup(ifname=interface.name)[0]
Expand Down
33 changes: 23 additions & 10 deletions octavia/amphorae/backends/utils/interface_file.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,22 +227,28 @@ def __init__(self, name, mtu, vips, vrrp_info, fixed_ips, topology,
fixed_ip.get('host_routes', []))
self.routes.extend(host_routes)

if is_sriov:
sriov_param = ' sriov'
else:
sriov_param = ''

for ip_v in ip_versions:
self.scripts[consts.IFACE_UP].append({
consts.COMMAND: (
"/usr/local/bin/lvs-masquerade.sh add {} {}".format(
'ipv6' if ip_v == 6 else 'ipv4', name))
"/usr/local/bin/lvs-masquerade.sh add {} {}{}".format(
'ipv6' if ip_v == 6 else 'ipv4', name, sriov_param))
})
self.scripts[consts.IFACE_DOWN].append({
consts.COMMAND: (
"/usr/local/bin/lvs-masquerade.sh delete {} {}".format(
'ipv6' if ip_v == 6 else 'ipv4', name))
"/usr/local/bin/lvs-masquerade.sh delete {} {}{}".format(
'ipv6' if ip_v == 6 else 'ipv4', name, sriov_param))
})


class PortInterfaceFile(InterfaceFile):
def __init__(self, name, mtu, fixed_ips):
super().__init__(name, if_type=consts.BACKEND, mtu=mtu)
def __init__(self, name, mtu, fixed_ips, is_sriov=False):
super().__init__(name, if_type=consts.BACKEND, mtu=mtu,
is_sriov=is_sriov)

if fixed_ips:
ip_versions = set()
Expand Down Expand Up @@ -271,14 +277,21 @@ def __init__(self, name, mtu, fixed_ips):
consts.IPV6AUTO: True
})

if is_sriov:
sriov_param = ' sriov'
else:
sriov_param = ''

for ip_version in ip_versions:
self.scripts[consts.IFACE_UP].append({
consts.COMMAND: (
"/usr/local/bin/lvs-masquerade.sh add {} {}".format(
'ipv6' if ip_version == 6 else 'ipv4', name))
"/usr/local/bin/lvs-masquerade.sh add {} {}{}".format(
'ipv6' if ip_version == 6 else 'ipv4', name,
sriov_param))
})
self.scripts[consts.IFACE_DOWN].append({
consts.COMMAND: (
"/usr/local/bin/lvs-masquerade.sh delete {} {}".format(
'ipv6' if ip_version == 6 else 'ipv4', name))
"/usr/local/bin/lvs-masquerade.sh delete {} {}{}".format(
'ipv6' if ip_version == 6 else 'ipv4', name,
sriov_param))
})
44 changes: 33 additions & 11 deletions octavia/amphorae/backends/utils/nftable_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,16 +26,26 @@
LOG = logging.getLogger(__name__)


def write_nftable_vip_rules_file(interface_name, rules):
def write_nftable_rules_file(interface_name, rules):
flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
# mode 00600
mode = stat.S_IRUSR | stat.S_IWUSR

# Create some strings shared on both code paths
table_string = f'table {consts.NFT_FAMILY} {consts.NFT_VIP_TABLE} {{\n'
chain_string = f' chain {consts.NFT_VIP_CHAIN} {{\n'
hook_string = (f' type filter hook ingress device {interface_name} '
f'priority {consts.NFT_SRIOV_PRIORITY}; policy drop;\n')
table_string = f'table {consts.NFT_FAMILY} {consts.NFT_TABLE} {{\n'
chain_string = f' chain {consts.NFT_CHAIN} {{\n'
vip_chain_string = f' chain {consts.NFT_VIP_CHAIN} {{\n'
hook_string = (' type filter hook input priority filter; '
'policy drop;\n')

# Conntrack is used to allow flow return traffic
conntrack_string = (' ct state vmap { established : accept, '
'related : accept, invalid : drop }\n')

# Allow loopback traffic on the loopback interface, no where else
loopback_string = ' iif lo accept\n'
loopback_addr_string = ' ip saddr 127.0.0.0/8 drop\n'
loopback_ipv6_addr_string = ' ip6 saddr ::1 drop\n'

# Allow ICMP destination unreachable for PMTUD
icmp_string = ' icmp type destination-unreachable accept\n'
Expand All @@ -47,38 +57,50 @@ def write_nftable_vip_rules_file(interface_name, rules):
dhcp_string = ' udp sport 67 udp dport 68 accept\n'
dhcpv6_string = ' udp sport 547 udp dport 546 accept\n'

# If the packet came in on the VIP interface, goto the VIP rules chain
vip_interface_goto_string = (
f' iifname {consts.NETNS_PRIMARY_INTERFACE} '
f'goto {consts.NFT_VIP_CHAIN}\n')

# Check if an existing rules file exists or we be need to create an
# "drop all" file with no rules except for VRRP. If it exists, we should
# not overwrite it here as it could be a reboot unless we were passed new
# rules.
if os.path.isfile(consts.NFT_VIP_RULES_FILE):
if os.path.isfile(consts.NFT_RULES_FILE):
if not rules:
return
with os.fdopen(
os.open(consts.NFT_VIP_RULES_FILE, flags, mode), 'w') as file:
os.open(consts.NFT_RULES_FILE, flags, mode), 'w') as file:
# Clear the existing rules in the kernel
# Note: The "nft -f" method is atomic, so clearing the rules will
# not leave the amphora exposed.
# Create and delete the table to not get errors if the table does
# not exist yet.
file.write(f'table {consts.NFT_FAMILY} {consts.NFT_VIP_TABLE} '
file.write(f'table {consts.NFT_FAMILY} {consts.NFT_TABLE} '
'{}\n')
file.write(f'delete table {consts.NFT_FAMILY} '
f'{consts.NFT_VIP_TABLE}\n')
f'{consts.NFT_TABLE}\n')
file.write(table_string)
file.write(chain_string)
file.write(hook_string)
file.write(conntrack_string)
file.write(loopback_string)
file.write(loopback_addr_string)
file.write(loopback_ipv6_addr_string)
file.write(icmp_string)
file.write(icmpv6_string)
file.write(dhcp_string)
file.write(dhcpv6_string)
file.write(vip_interface_goto_string)
file.write(' }\n') # close the chain
file.write(vip_chain_string)
for rule in rules:
file.write(f' {_build_rule_cmd(rule)}\n')
file.write(' }\n') # close the chain
file.write('}\n') # close the table
else: # No existing rules, create the "drop all" base rules
with os.fdopen(
os.open(consts.NFT_VIP_RULES_FILE, flags, mode), 'w') as file:
os.open(consts.NFT_RULES_FILE, flags, mode), 'w') as file:
file.write(table_string)
file.write(chain_string)
file.write(hook_string)
Expand Down Expand Up @@ -113,7 +135,7 @@ def _build_rule_cmd(rule):


def load_nftables_file():
cmd = [consts.NFT_CMD, '-o', '-f', consts.NFT_VIP_RULES_FILE]
cmd = [consts.NFT_CMD, '-o', '-f', consts.NFT_RULES_FILE]
try:
with network_namespace.NetworkNamespace(consts.AMPHORA_NAMESPACE):
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
Expand Down
Loading

0 comments on commit 684a8c8

Please sign in to comment.