ipv6.azcli

############################################################################
# Created by Jose Moreno
# March 2020
#
# The script creates a Linux VM and a VMSS with IPv4 and updates it with IPv6
# It installs Apache inside of the VMs to test connectivity
#
# Example of tests you can run with these commands:
# * Create a VM/VMSS on an IPv6-only subnet
# * Create a VM/VMSS with IPv4, and upgrade it to IPv6
# * Assign public IPv6 addresses to IPv4 VM/VMSS
#
# References:
# https://docs.microsoft.com/azure/virtual-network/ipv6-overview
# https://docs.microsoft.com/azure/virtual-network/virtual-network-ipv4-ipv6-dual-stack-standard-load-balancer-cli
# https://docs.microsoft.com/azure/virtual-network/ipv6-add-to-existing-vnet-cli
# https://docs.microsoft.com/azure/virtual-network/ipv6-virtual-machine-scale-set
# Old: https://docs.microsoft.com/azure/load-balancer/load-balancer-ipv6-overview
#
############################################################################

# IPv6 vnet in preview
az feature register --name AllowIPv6VirtualNetwork --namespace Microsoft.Network
az feature register --name AllowIPv6CAOnStandardLB --namespace Microsoft.Network
# Wait for both features to be registered
az feature show --name AllowIPv6VirtualNetwork --namespace Microsoft.Network
az feature show --name AllowIPv6CAOnStandardLB --namespace Microsoft.Network
# Register RP
az provider register --namespace Microsoft.Network

# Initialization
rg=ipv6
location=eastus
vnet_name=vnet
vnet_prefix_v4=192.168.0.0/16
vnet_prefix_v6=ace:cab:deca::/48
ipv4_subnet_name=ipv4-only
ipv4_subnet_prefix=192.168.10.0/24
ipv6_subnet_name=ipv6-only
ipv6_subnet_prefix=ace:cab:deca:11::/64
ds_subnet_name=dual-stack
ds_subnet_prefix_ipv4=192.168.12.0/24
ds_subnet_prefix_ipv6=ace:cab:deca:12::/64
vmss_subnet_prefix_v4=192.168.1.0/24
cloudinit_filename='/tmp/cloudinit-apache2.txt'
nsg_name=vmss-nsg
vmss_subnet_name=vmss
vmss_subnet_prefix_v4=192.168.1.0/24
vmss_subnet_prefix_v6=ace:cab:deca:1::/64
vmss_name=myvmss
vmss_sku=Standard_B1s
vmss_vm_count=1
vmss_pipv4_name=vmss_pipv4
vmss_pipv6_name=vmss_pipv6
vmss_lb_sku=Standard
vmss_lb_natpool=natpool
vmss_lb_name=vmss-lb
vm_subnet_name=vm
vm_subnet_prefix_v4=192.168.2.0/24
vm_subnet_prefix_v6=ace:cab:deca:2::/64
vm_static_ipv6=ace:cab:deca:2::4
vm_name=testvm
vm_sku=Standard_B1s
vm_pipv4_name=vm_pipv4
vm_pipv6_name=vm_pipv6
nva_subnet_name=nva
nva_subnet_prefix_v4=192.168.50.0/24
nva_subnet_prefix_v6=ace:cab:deca:50::/64
nva_name=nva
nva_cloudinit_filename=cloudinit-iptables.txt
nva_sku=Standard_B1s
nva_pipv4_name=nva_pipv4
nva_pipv6_name=nva_pipv6
nva_lb_sku=Standard
nva_lb_ext_name=nva-lb-ext
nva_lb_ext_pipv4=nva-lb-pipv4
nva_lb_ext_pipv6=nva-lb-pipv6
nva_lb_ext_frontend_v4_name=frontend-v4
nva_lb_ext_frontend_v6_name=frontend-v6
nva_lb_ext_backend_v4_name=backend-v4
nva_lb_ext_backend_v6_name=backend-v6
nva_lb_int_name=nva-lb-int
nva_lb_int_frontend_ipv4=192.168.50.100
nva_lb_int_frontend_ipv6=ace:cab:deca:50::100
nva_lb_int_frontend_v4_name=frontend-v4
nva_lb_int_frontend_v6_name=frontend-v6
nva_lb_int_backend_v4_name=backend-v4
nva_lb_int_backend_v6_name=backend-v6
lb_probe_name=SSHprobe
lb_probe_port=22

# Create RG
az group create -n $rg -l $location

# Create VM/VMSS vnet/subnets
az network vnet create -g $rg -n $vnet_name --address-prefix $vnet_prefix_v4
az network vnet subnet create -g $rg --vnet-name $vnet_name -n $vmss_subnet_name --address-prefixes $vmss_subnet_prefix_v4
az network vnet subnet create -g $rg --vnet-name $vnet_name -n $vm_subnet_name --address-prefixes $vm_subnet_prefix_v4

# Optionally add the IPv6 prefixes to existing IPv4-only
az network vnet update -g $rg -n $vnet_name --address-prefixes $vnet_prefix_v4 $vnet_prefix_v6
az network vnet subnet update -g $rg --vnet-name $vnet_name -n $vmss_subnet_name --address-prefixes $vmss_subnet_prefix_v4 $vmss_subnet_prefix_v6
az network vnet subnet update -g $rg --vnet-name $vnet_name -n $vm_subnet_name --address-prefixes $vm_subnet_prefix_v4 $vm_subnet_prefix_v6

# Alternatively create dual-stack vnets/subnets directly
# az network vnet create -g $rg -n $vnet_name --address-prefixes $vnet_prefix_v4 $vnet_prefix_v6
# az network vnet subnet create -g $rg --vnet-name $vnet_name -n $vmss_subnet_name --address-prefixes $vmss_subnet_prefix_v4 $vmss_subnet_prefix_v6
# az network vnet subnet create -g $rg --vnet-name $vnet_name -n $vm_subnet_name --address-prefixes $vm_subnet_prefix $vm_subnet_prefix_v6

# Or create IPv6-only vnets
# az network vnet create -g $rg -n $vnet_name --address-prefix $vnet_prefix_v6
# az network vnet subnet create -g $rg --vnet-name $vnet_name -n $vmss_subnet_name --address-prefixes $vmss_subnet_prefix_v6
# az network vnet subnet create -g $rg --vnet-name $vnet_name -n $vm_subnet_name --address-prefixes $vm_subnet_prefix_v6

# Optional: Assuming a dual stack vnet, create specific subnets for IPv4-only and IPv6-only
# These subnets are for testing, additionally to the NVA/VM/VMSS subnets
az network vnet subnet create -g $rg --vnet-name $vnet_name -n $ipv4_subnet_name --address-prefix $ipv4_subnet_prefix
az network vnet subnet create -g $rg --vnet-name $vnet_name -n $ipv6_subnet_name --address-prefix $ipv6_subnet_prefix
az network vnet subnet create -g $rg --vnet-name $vnet_name -n $ds_subnet_name --address-prefixes $ds_subnet_prefix_ipv4 $ds_subnet_prefix_ipv6

# Create PIPs (all created as standard SKUs)
az network public-ip create -n $vmss_pipv4_name -g $rg --sku Standard --allocation-method static --version IPv4
az network public-ip create -n $vmss_pipv6_name -g $rg --sku Standard --allocation-method static --version IPv6
az network public-ip create -n $vm_pipv4_name -g $rg --sku Standard --allocation-method static --version IPv4
az network public-ip create -n $vm_pipv6_name -g $rg --sku Standard --allocation-method static --version IPv6
vm_pipv4_ip=$(az network public-ip show -n $vm_pipv4_name -g $rg --query ipAddress -o tsv) && echo $vm_pipv4_ip
vm_pipv6_ip=$(az network public-ip show -n $vm_pipv6_name -g $rg --query ipAddress -o tsv) && echo $vm_pipv6_ip

# Create NSG allowing inbound SSH and 80 to the ipv6 prefix
# Note that specifying all addresses as '*' defaults to IPv4 (rule #510). In order to allow IPv6 you need to specify at least one IPv6 range (like in rule #520)
az network nsg create -n $nsg_name -g $rg
az network nsg rule create -n ssh --nsg-name $nsg_name -g $rg --priority 500 --destination-port-ranges 22 --access Allow --protocol Tcp
az network nsg rule create -n webstar --nsg-name $nsg_name -g $rg --priority 510 --destination-address-prefixes '*' --destination-port-ranges 80 --access Allow --protocol Tcp
az network nsg rule create -n webv6 --nsg-name $nsg_name -g $rg --priority 520 --destination-address-prefixes $vnet_prefix_v6 --destination-port-ranges 80 --access Allow --protocol Tcp
az network nsg rule list --nsg-name $nsg_name -g $rg -o table

# Optionally: create additional vnet to peer
peer_vnet_name=peer
peer_vnet_prefix_v4=172.16.0.0/16
peer_vnet_prefix_v6=ace:cab:decb::/48
peer_subnet_name=test-subnet
peer_subnet_prefix_v4=172.16.1.0/24
peer_subnet_prefix_v6=ace:cab:decb:1::/64
az network vnet create -g $rg -n $peer_vnet_name --address-prefixes $peer_vnet_prefix_v4 $peer_vnet_prefix_v6
az network vnet subnet create -g $rg --vnet-name $peer_vnet_name -n $peer_subnet_name --address-prefixes $peer_subnet_prefix_v4 $peer_subnet_prefix_v6
az network vnet peering create -n vnet2peer -g $rg --vnet-name $vnet_name --remote-vnet $peer_vnet_name --allow-vnet-access --allow-forwarded-traffic
az network vnet peering create -n peer2vnet -g $rg --vnet-name $peer_vnet_name --remote-vnet $vnet_name --allow-vnet-access --allow-forwarded-traffic

# Create cloudinit file
cat << 'EOF' > $cloudinit_filename
#cloud-config
runcmd:
  - apt-get update
  - apt-get install -y apache2
EOF


########
#  VM  #
########

# Create VM (ipv4-only with no PIP, ipv6 added next with IPv6 PIP)
az vm create -n $vm_name -g $rg --image UbuntuLTS --generate-ssh-keys --size $vm_sku --custom-data $cloudinit_filename \
   --vnet-name $vnet_name --subnet $vm_subnet_name --nsg $nsg_name --public-ip-address ""
nic_id=$(az vm show -n $vm_name -g $rg --query 'networkProfile.networkInterfaces[0].id' -o tsv)
nic_name=$(echo $nic_id | cut -d/ -f 9)
az network nic ip-config create -n ipv6Config --nic-name $nic_name -g $rg --vnet-name $vnet_name --subnet $vm_subnet_name --private-ip-address-version IPv6 --public-ip-address $vm_pipv6_name
az vm list-ip-addresses -n $vm_name -g $rg -o table
az network nic list-effective-nsg -n $nic_name -g $rg

# Optionally, add PIPv4 (you can use this method to add a PIP to the correct ipconfig)
ipconfig_version=IPv4
ipconfig_name=$(az network nic ip-config list --nic-name $nic_name -g $rg --query "[?privateIpAddressVersion=='${ipconfig_version}'].{Name:name}" -o tsv)
az network nic ip-config update -n $ipconfig_name --nic-name $nic_name -g $rg --public-ip-address $vm_pipv4_name
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no "$vm_pipv4_ip" "ip a"

# Create VM (ipv6-only) -> Does NOT work
# With IPv6 PIP and static private IPv6 does not work
az vm create -n $vm_name -g $rg --image UbuntuLTS --generate-ssh-keys --size $vm_sku --custom-data $cloudinit_filename \
   --vnet-name $vnet_name --subnet $ipv6_subnet_name --public-ip-address $vm_pipv6_name --private-ip-address "ace:cab:deca:11::4" \
   --nsg $nsg_name
# Without PIP or static private IP on an IPv6-only subnet will not work
az vm create -n $vm_name -g $rg --image UbuntuLTS --generate-ssh-keys --size $vm_sku --custom-data $cloudinit_filename \
   --vnet-name $vnet_name --subnet $ipv6_subnet_name \
   --nsg $nsg_name
# Creating just a NIC with IPv6-only will not work
az network nic create -n ${vm_name}-nic -g $rg --vnet-name $vnet_name --subnet $ipv6_subnet_name --private-ip-address-version IPv6 --public-ip-address $vm_pipv6_name
nic_id=$()
az vm create -n $vm_name -g $rg --image UbuntuLTS --generate-ssh-keys --size $vm_sku --custom-data $cloudinit_filename \
   --nics ${vm_name}-nic \
   --nsg $nsg_name

# Possible errors:
# If not specifying public/private IPs in an IPv6-only vnet:
#   "Private IP version IPv4 of IP configuration /blah/ipconfigtestvm is not permitted in the subnet /blah/ipv6-only which does not have addressprefixes of version IPv4."
# If creating a VM in a dual-stack vnet without specifying the private IP, and trying to associate a public IPv6 address:
#   "IP version for publicIPAddress '/blah/vm_pipv6', version 'IPv6' and network interface ipConfiguration '/blah/ipconfigtestvm', version 'IPv4' must match."
# If creating a VM in a dual-stack vnet specifying a static private IPv6:
#   "Static private IP ace:cab:deca:2::4 on IP configuration /blah/ipconfigtestvm is not a valid IPv4 IP address."
# Even in an ipv6-only vnet, defining a static IPv6 address, it tries to allocate as if being an IPv4:
#   "Static private IP ace:cab:deca:11::4 on IP configuration /blah/ipconfigtestvm is not a valid IPv4 IP address."
# No IPv6-only IP configs supported (when creating a NIC)
#   "At least one IPv4 ipConfiguration is required for an IPv6 ipConfiguration on the network interface '/blah/testvm-nic'."

#########
#  NVA  #
#########
az network vnet subnet create -g $rg --vnet-name $vnet_name -n $nva_subnet_name --address-prefixes $nva_subnet_prefix_v4 $nva_subnet_prefix_v6
az network public-ip create -n $nva_pipv4_name -g $rg --sku Standard --allocation-method static --version IPv4
az network public-ip create -n $nva_pipv6_name -g $rg --sku Standard --allocation-method static --version IPv6
nva_pipv4_ip=$(az network public-ip show -n $nva_pipv4_name -g $rg --query ipAddress -o tsv)
nva_pipv6_ip=$(az network public-ip show -n $nva_pipv6_name -g $rg --query ipAddress -o tsv)
# VM
az vm create -n $nva_name -g $rg --image UbuntuLTS --generate-ssh-keys --size $vm_sku --custom-data $nva_cloudinit_filename \
   --vnet-name $vnet_name --subnet $nva_subnet_name --nsg $nsg_name --public-ip-address $nva_pipv4_name
nva_nic_id=$(az vm show -n $nva_name -g $rg --query 'networkProfile.networkInterfaces[0].id' -o tsv)
nva_nic_name=$(echo $nva_nic_id | cut -d/ -f 9)
az network nic ip-config create -n ipv6Config --nic-name $nva_nic_name -g $rg --vnet-name $vnet_name --subnet $vm_subnet_name --private-ip-address-version IPv6 --public-ip-address $nva_pipv6_name
az network nic update -n $nva_nic_name -g $rg --ip-forwarding true
# Public ALB
az network lb create -g $rg -n $nva_lb_ext_name --sku $nva_lb_sku --vnet-name $vnet_name \
   --frontend-ip-name $nva_lb_ext_frontend_v6_name --public-ip-address $nva_lb_ext_pipv6 --backend-pool-name $nva_lb_ext_backend_v6_name
az network lb probe create -g $rg --lb-name $nva_lb_ext_name -n $lb_probe_name --protocol tcp --port $lb_probe_port
az network lb rule create -n HTTP -g $rg --lb-name $nva_lb_ext_name --protocol Tcp --frontend-port 80 --backend-port 80 \
   --frontend-ip-name $nva_lb_ext_frontend_v6_name --backend-pool-name $nva_lb_ext_backend_v6_name --probe-name $lb_probe_name
nva_lb_ext_backend_v6_id=$(az network lb address-pool show -n $nva_lb_ext_backend_v6_name --lb-name $nva_lb_ext_name -g $rg --query id -o tsv)
# Internal ALB
az network lb create -g $rg -n $nva_lb_int_name --sku $nva_lb_sku --vnet-name $vnet_name --subnet $nva_subnet_name \
   --frontend-ip-name $nva_lb_int_frontend_v6_name  --private-ip-address-version IPv6 --private-ip-address $nva_lb_int_frontend_ipv6 \
   --backend-pool-name $nva_lb_int_backend_v6_name
az network lb probe create -g $rg --lb-name $nva_lb_int_name -n $lb_probe_name --protocol tcp --port $lb_probe_port
az network lb rule create -n AllPorts -g $rg --lb-name $nva_lb_int_name --protocol All --frontend-port 0 --backend-port 0 \
   --frontend-ip-name $nva_lb_int_frontend_v6_name --backend-pool-name $nva_lb_int_backend_v6_name --probe-name $lb_probe_name
nva_lb_int_backend_v6_id=$(az network lb address-pool show -n $nva_lb_int_backend_v6_name --lb-name $nva_lb_int_name -g $rg --query id -o tsv)
# Associate NVA to IPv6 address pools
az network nic ip-config update -n ipv6Config --nic-name $nva_nic_name -g $rg \
   --lb-address-pools $nva_lb_ext_backend_v6_id $nva_lb_int_backend_v6_id
# Default IPv6 route for the VM subnet 
az network route-table create -n vm -g $rg
az network route-table route create -n default-v6 --route-table vm -g $rg \
   --address-prefix '::/0' --next-hop-type VirtualAppliance --next-hop-ip-address $nva_lb_int_frontend_ipv6
az network vnet subnet update -n $vm_subnet_name --vnet-name $vnet_name -g $rg --route-table vm

# Test Outgoing traffic from VM
ssh $vm_pipv4_ip "curl -s6 ifconfig.co"
echo "You should be seeing the NVA's IP address: $nva_pipv6_ip"

# Verify NVA config
ssh $nva_pipv4_ip "sysctl net.ipv6.conf.all.forwarding"
sleep 2
ssh $nva_pipv4_ip "sudo iptables -t nat -L"
sleep 2
ssh $nva_pipv4_ip "ip -6 route"
# This line will be ignored after an SSH command (??)
ssh $vm_pipv4_ip "ip -6 route"


########
# AzFW #
########

# TBD

########
# VMSS #
########

# Create VMSS
# VMSS creation like this will fail if vnet is IPv6-only
az vmss create -n $vmss_name -g $rg -l $location --image UbuntuLTS --generate-ssh-keys \
    --vm-sku $vmss_sku --instance-count $vmss_vm_count --custom-data $cloudinit_filename \
    --vnet-name $vnet_name --subnet $vmss_subnet_name --nsg $nsg_name \
    --lb-sku $vmss_lb_sku --lb $vmss_lb_name --lb-nat-pool-name $vmss_lb_natpool --public-ip-address $vmss_pipv4_name

# Optionally, create VMSS without LB
# The AZs are another trick to force Standard PIPs/ALB
az vmss create -n $vmss_name -g $rg -l $location --image UbuntuLTS --generate-ssh-keys \
    --vm-sku $vmss_sku --instance-count $vmss_vm_count --custom-data $cloudinit_filename \
    --vnet-name $vnet_name --subnet $vmss_subnet_name --nsg $nsg_name \
    --lb "" --public-ip-per-vm --zones 1 2 3 --no-wait

# Combining Standard LB with ILPIP will not work (SKU mismatch, because the created IPv4 ILPIPs are basic)
az vmss create -n $vmss_name -g $rg -l $location --image UbuntuLTS --generate-ssh-keys \
    --vm-sku $vmss_sku --instance-count $vmss_vm_count --custom-data $cloudinit_filename \
    --vnet-name $vnet_name --subnet $vmss_subnet_name --nsg $nsg_name \
    --lb-sku $vmss_lb_sku --lb $vmss_lb_name --lb-nat-pool-name $vmss_lb_natpool --public-ip-per-vm

# Add IPv6 ALB frontend, probe, backend and LB rule
# You can do this on a VMSS with a standard ALB, or without any ALB
backend_name_v4=$(az network lb address-pool list --lb-name $vmss_lb_name -g $rg --query '[0].name' -o tsv)
backend_name_v6=backendv6
frontend_name_v4=$(az network lb frontend-ip list --lb-name $vmss_lb_name -g $rg --query '[0].name' -o tsv)
frontend_name_v6=frontend_v6
rule_name_v4=$(az network lb rule list --lb-name $vmss_lb_name -g $rg --query '[0].name' -o tsv)
rule_name_v6=webv6
rule_frontend_port_v6=8080
rule_backend_port_v6=80
az network lb frontend-ip create -n $frontend_name_v6 -g $rg --lb-name $vmss_lb_name --public-ip-address $vmss_pipv6_name
az network lb probe create -n $lb_probe_name -g $rg --lb-name $vmss_lb_name --protocol tcp --port $lb_probe_port

# If you tried to associate the backend to the NICs here, you would get an error when using the same backend pool for IPv4 and IPv6 frontends:
# "Frontend ipConfiguration '/blah/frontend_v6' referring to PublicIp with PublicIpAddressVersion 'IPv6' 
#   does not match with PrivateIpAddressVersion 'IPv4' referenced by backend ipConfiguration '/blah/myvms74b2IPConfig' 
#   for the load balancer rule '/blah/SSHv6'"

# Create a new backend pool
az network lb address-pool create -n $backend_name_v6 -g $rg --lb-name $vmss_lb_name
az network lb rule create -n $rule_name_v6 -g $rg --lb-name $vmss_lb_name --frontend-ip-name $frontend_name_v6 --protocol Tcp --frontend-port $rule_frontend_port_v6 --backend-port $rule_backend_port_v6 --probe-name $lb_probe_name --backend-pool-name $backend_name_v6

# If you tried to associate the backend to the NICs here, you would get an error when configuring an IPv6 backend pools in an IPv4 ipconfig in the VMSS:
# "Cannot specify load balancer inbound rule '/blah/SSHv6' connected to IPv6 publicIPAddress '/blah/vmss_pipv6'
#   for an IPv4 network interface ipConfiguration 'myvms74b2IPConfig' of VM Scale Set '/blah/myvmss'
#   as the publicIpAddressVersion of the publicIPAddress does not match with the privateIpAddressVersion of the network interface ipConfiguration"

# Add new ip config to the VMSS
backend_id_v4=$(az network lb address-pool show -n $backend_name_v4 -g $rg --lb-name $vmss_lb_name --query id -o tsv)
backend_id_v6=$(az network lb address-pool show -n $backend_name_v6 -g $rg --lb-name $vmss_lb_name --query id -o tsv)
vmss_subnet_id=$(az network vnet subnet show -n $vmss_subnet_name --vnet-name $vnet_name -g $rg --query id -o tsv)

# If you tried to associate the backend to the NICs here, you would get an error because the existing rule uses a different probe:
# "The network interface 'myvms74b2Nic' of VM Scale Set '/blah/myvmss' has an IPv6 rule with load balancer probe '/blah/SSHprobe' 
#   but it does not have the same probe on an IPv4 rule"

# Update previous IPv4 rule to use the same probe
az network lb rule update -n $rule_name_v4 --lb-name $vmss_lb_name -g $rg --probe-name $lb_probe_name

# Create new ip config with backend association
ipconfig_v6="{
    \"name\": \"ipconfigv6\",
    \"subnet\": { \"id\": \"$vmss_subnet_id\" },
    \"privateIPAddressVersion\":\"IPv6\",
    \"loadBalancerBackendAddressPools\": [{ \"id\": \"$backend_id_v6\" }]}"
az vmss update -g $rg -n $vmss_name --add virtualMachineProfile.networkProfile.networkInterfaceConfigurations\[0\].ipConfigurations $ipconfig_v6
az vmss update-instances -g $rg --name $vmss_name --instance-ids "*"

# Alternatively, add a new IP config with ILPIP v6 (instead of the ALB association)
vmss_subnet_id=$(az network vnet subnet show -n $vmss_subnet_name --vnet-name $vnet_name -g $rg --query id -o tsv)
ipconfig_v6="{
    \"name\": \"ipconfigv6\",
    \"subnet\": { \"id\": \"$vmss_subnet_id\" },
    \"privateIPAddressVersion\":\"IPv6\",
    \"publicIpAddressConfiguration\": { \"name\": \"instancepublicipv6\", \"publicIPAddressVersion\": \"IPv6\" }}"
az vmss update -g $rg -n $vmss_name --add virtualMachineProfile.networkProfile.networkInterfaceConfigurations\[0\].ipConfigurations $ipconfig_v6
az vmss show -g $rg -n $vmss_name --query 'virtualMachineProfile.networkProfile.networkInterfaceConfigurations[0].ipConfigurations[]'
az vmss update-instances -g $rg --name $vmss_name --instance-ids "*"

# Error because IPv4 PIPs deployed with Basic SKU
# Deployment failed. Correlation ID: blah. Different basic sku and standard sku load balancer or public Ip resources in availability set is not allowed 
#   /blah|instancepublicipv6 and /blah|instancepublicip.

# Optional: modify previous IPv4 config
az vmss show -g $rg -n $vmss_name --query 'virtualMachineProfile.networkProfile.networkInterfaceConfigurations[0].ipConfigurations'
ipconfig_v4="{
    \"name\": \"ipconfigv4\",
    \"subnet\": { \"id\": \"$vmss_subnet_id\" },
    \"privateIPAddressVersion\":\"IPv4\",
    \"publicIpAddressConfiguration\": { \"name\": \"instancepublicipv4\", \"publicIPAddressVersion\": \"IPv4\", \"sku\":{\"name\": \"standard\"}}}"
az vmss update -g $rg -n $vmss_name --set virtualMachineProfile.networkProfile.networkInterfaceConfigurations\[0\].ipConfigurations\[0\] $ipconfig_v4

# Optional: add a new NIC with a 1ary IPv4 ipconfig, a 2ary ipv6 ipconfig, and an ILPIP in the ipv6 ipconfig
# not working :(
az vmss show -g $rg -n $vmss_name --query 'virtualMachineProfile.networkProfile.networkInterfaceConfigurations'
ipconfig_v4="{
        \"name\": \"ipv4config\",
        \"subnet\": { \"id\": \"$vmss_subnet_id\" },
        \"privateIPAddressVersion\":\"IP${version}\"}"
# Removed the ILPIP for testing purposes:
ipconfig_v6="{
        \"name\": \"ipconfigv6\",
        \"subnet\": { \"id\": \"$vmss_subnet_id\" },
        \"privateIPAddressVersion\":\"IPv6\"}"
nsg_config="{
        \"id\": \"/subscriptions/e7da9914-9b05-4891-893c-546cb7b0422e/resourceGroups/$rg/providers/Microsoft.Network/networkSecurityGroups/$nsg_name\",
        \"resourceGroup\": \"$rg\" }"
nic_2="{
    \"name\": \"vmssnic2\",
    \"dnsSettings\": { \"dnsServers\": [] },
    \"enableAcceleratedNetworking\": false,
    \"enableIpForwarding\": false,
    \"ipConfigurations\": [ $ipconfig_v4, $ipconfig_v6 ],
    \"networkSecurityGroup\": $nsg_config ,
    \"primary\": false }"
az vmss deallocate -n $vmss_name -g $rg
az vmss update -g $rg -n $vmss_name --add virtualMachineProfile.networkProfile.networkInterfaceConfigurations $nic_2
# Error if ILPIP in 2ary ipv6 ipconfig(?????):
# Cannot specify public ip address for a secondary nic IPv6 ipConfiguration 'ipconfigv6' of VM Scale Set '/blah/myvmss' since the scale set is in an IPv6 vnet.
# Add ILPIP to 2ary config - NOT working either
ipv6_ilpip="{ \"name\": \"ilpipv6\", \"publicIPAddressVersion\": \"IPv6\" }"
az vmss show -g $rg -n $vmss_name --query 'virtualMachineProfile.networkProfile.networkInterfaceConfigurations[1].ipConfigurations[1]'
az vmss show -g $rg -n $vmss_name --query 'virtualMachineProfile.networkProfile.networkInterfaceConfigurations[1].ipConfigurations[1].publicIpAddressConfiguration'
az vmss update -g $rg -n $vmss_name --set 'virtualMachineProfile.networkProfile.networkInterfaceConfigurations[1].ipConfigurations[1].publicIpAddressConfiguration'=$ipv6_ilpip
# Error if ILPIP in 2ary ipv6 ipconfig(?????):
# Cannot specify public ip address for a secondary nic IPv6 ipConfiguration 'ipconfigv6' of VM Scale Set '/blah/myvmss' since the scale set is in an IPv6 vnet.

# az vmss update -g $rg -n $vmss_name --remove virtualMachineProfile.networkProfile.networkInterfaceConfigurations 1
az vmss show -g $rg -n $vmss_name --query 'virtualMachineProfile.networkProfile.networkInterfaceConfigurations'
az vmss update-instances -g $rg --name $vmss_name --instance-ids "*"
az vmss start -n $vmss_name -g $rg

# Optional: replace the 1ary ipv4 nic with an ipv6 one
az vmss show -g $rg -n $vmss_name --query 'virtualMachineProfile.networkProfile.networkInterfaceConfigurations[0].ipConfigurations[0]'
ipconfig_v6="{
    \"name\": \"ipconfigv6\",
    \"subnet\": { \"id\": \"$vmss_subnet_id\" },
    \"privateIPAddressVersion\":\"IPv6\",
    \"publicIpAddressConfiguration\": { \"name\": \"instancepublicipv6\", \"publicIPAddressVersion\": \"IPv6\" }}"
az vmss update -g $rg -n $vmss_name --set 'virtualMachineProfile.networkProfile.networkInterfaceConfigurations[0].ipConfigurations[0]'=$ipconfig_v6
az vmss update -g $rg -n $vmss_name \
--set 'virtualMachineProfile.networkProfile.networkInterfaceConfigurations[0].ipConfigurations[0].privateIPAddressVersion'=IPv6 \
--set 'virtualMachineProfile.networkProfile.networkInterfaceConfigurations[0].ipConfigurations[0].publicIpAddressConfiguration.publicIPAddressVersion'=IPv6

###############
#   Cleanup   #
###############

# Stop VMSS and VMs
az vmss stop -n $vmss_name -g $rg --no-wait
az vm deallocate -n $vm_name -g $rg --no-wait
az vm deallocate -n $nva_name -g $rg --no-wait

# Alternatively, delete VMSS and VM
az vmss delete -n $vmss_name -g $rg --no-wait
az vm delete -n $vm_name -g $rg -y --no-wait
az vm delete -n $nva_name -g $rg -y --no-wait

# Or more drastically, delete whole RG
# az group delete -n $rg -y --no-wait

###############
# Diagnostics #
###############

# Verify vnet prefixes
az network vnet list -g $rg -o table
az network vnet list -g $rg --query '[].{Name:name,Prefix:addressPrefix,Prefixes1:addressPrefixes[0],Prefixes2:addressPrefixes[1],Prefixes3:addressPrefixes[2]}' -o table

# Verify subnet prefixes
az network vnet subnet list --vnet-name $vnet_name -g $rg -o table
az network vnet subnet list --vnet-name $vnet_name -g $rg --query '[].{Name:name,Prefix:addressPrefix,Prefixes1:addressPrefixes[0],Prefixes2:addressPrefixes[1],Prefixes3:addressPrefixes[2]}' -o table
az network vnet subnet show -n $vmss_subnet_name --vnet-name $vnet_name -g $rg --query '[addressPrefix,addressPrefixes]'
az network vnet subnet show -n $vm_subnet_name --vnet-name $vnet_name -g $rg --query '[addressPrefix,addressPrefixes]'

# Verify effective routes
az network nic show-effective-route-table -n $nic_name -g $rg -o table

# Verify public IP addresses
az network public-ip list -g $rg -o table

# Verify NSG rules
az network nsg rule list --nsg-name $nsg_name -g $rg -o table

# Verify ALB
az network public-ip list -g $rg -o table
az network lb list -g $rg -o table
az network lb frontend-ip list --lb-name $vmss_lb_name -g $rg -o table
az network lb inbound-nat-pool list --lb-name $vmss_lb_name -g $rg -o table
az network lb inbound-nat-rule list --lb-name $vmss_lb_name -g $rg -o table
az network lb rule list --lb-name $vmss_lb_name -g $rg
az network lb address-pool list --lb-name $vmss_lb_name -g $rg -o table
az network lb probe list --lb-name $vmss_lb_name -g $rg -o table

# Check ALB metrics
lb_id=$(az network lb show -n $vmss_lb_name -g $rg --query id -o tsv)
# Last hour, aggregated in 15min
az monitor metrics list --resource $lb_id --metric DipAvailability --interval 15m -o table
# Last 5 min
start_time=$(date --date='5 minutes ago' -u +"%Y-%m-%dT%H:%M:%SZ")
az monitor metrics list --resource $lb_id --metric DipAvailability --start-time $start_time --interval 1m -o table

# Verify VM and network config
az vm list -g $rg -o table
az vm list -g $rg -o table -d
az network nic ip-config list --nic-name $nic_name -g $rg -o table

# Verify VMSS network config
az vmss list -g $rg -o table
az vmss show -g $rg -n $vmss_name --query 'virtualMachineProfile.networkProfile.networkInterfaceConfigurations[0]'
az vmss show -g $rg -n $vmss_name --query 'virtualMachineProfile.networkProfile.networkInterfaceConfigurations[0].ipConfigurations[]'
az vmss list-instance-connection-info -g $rg -n $vmss_name -o table
az vmss list-instance-public-ips -g $rg -n $vmss_name -o table
az vmss list-instance-public-ips -g $rg -n $vmss_name --query '[].[name,ipAddress,sku.name]' -o tsv

# Cleanup
# az group delete -n $rg -y --no-wait