-
Notifications
You must be signed in to change notification settings - Fork 33
/
aks_appgw.azcli
343 lines (308 loc) · 12.2 KB
/
aks_appgw.azcli
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
#################################
# Created by Jose Moreno
# July 2020
#
# Some useful commands around AKS
#################################
# Variables
rg=akstest3
location=westeurope
keyvaultname=erjositoKeyvault
keyvault_rg=myKeyvault
wait_interval=5s
# AKS
aks_name=aks
aks_rbac=yes
aks_service_cidr=10.0.0.0/16
vmsize=Standard_B2ms
vm_size=Standard_B2ms
preview_version=no
# App Gateway
appgw_pip_name=appgw-pip
appgw_name=aksappgw
appgw_sku=Standard_v2
# Vnet
vnet_name=aksVnet
vnet_prefix=10.13.0.0/16
aks_subnet_name=aks
aks_subnet_prefix=10.13.76.0/24
vm_subnet_name=vm
vm_subnet_prefix=10.13.1.0/24
appgw_subnet_name=AppGateway
appgw_subnet_prefix=10.13.10.0/24
azfw_subnet_prefix=10.13.11.0/24
apim_subnet_prefix=10.13.12.0/24
db_subnet_prefix=10.13.50.0/24
akslb_subnet_prefix=10.13.77.0/24
arm_subnet_prefix=10.13.79.0/24
aci_subnet_prefix=10.13.100.0/24
# Other
logws_name=log$RANDOM
kv_name=erjositoKeyvault
acr_name=erjositoAcr
####################
# Helper functions #
####################
# Wait for GW to be created
function wait_until_finished {
wait_interval=15
resource_id=$1
resource_name=$(echo $resource_id | cut -d/ -f 9)
echo "Waiting for resource $resource_name to finish provisioning..."
start_time=`date +%s`
state=$(az resource show --id $resource_id --query properties.provisioningState -o tsv)
until [[ "$state" == "Succeeded" ]] || [[ "$state" == "Failed" ]] || [[ -z "$state" ]]
do
sleep $wait_interval
state=$(az resource show --id $resource_id --query properties.provisioningState -o tsv)
done
if [[ -z "$state" ]]
then
echo "Something really bad happened..."
else
run_time=$(expr `date +%s` - $start_time)
((minutes=${run_time}/60))
((seconds=${run_time}%60))
echo "Resource $resource_name provisioning state is $state, wait time $minutes minutes and $seconds seconds"
fi
}
##################
# Enable feature #
##################
# This will not be required after preview any more
# az feature register --name AKS-IngressApplicationGatewayAddon --namespace microsoft.containerservice
# state=$(az feature list -o table --query "[?contains(name, 'microsoft.containerservice/AKS-IngressApplicationGatewayAddon')].properties.state" -o tsv)
# echo "Waiting for feature to finish registering"
# wait_interval=15
# until [[ "$state" == "Succeeded" ]]
# do
# sleep $wait_interval
# state=$(az feature list -o table --query "[?contains(name, 'microsoft.containerservice/AKS-IngressApplicationGatewayAddon')].properties.state" -o tsv)
# done
# az provider register --namespace Microsoft.ContainerService
########
# Main #
########
# Create RG, LA workspace, vnet, AKS
az group create -n $rg -l $location
az monitor log-analytics workspace create -n $logws_name -g $rg
logws_id=$(az resource list -g $rg -n $logws_name --query '[].id' -o tsv)
acr_rg=$(az acr list -o tsv --query "[?name=='$acr_name'].resourceGroup")
acr_id=$(az acr show -n erjositoAcr -g $acr_rg --query id -o tsv)
az network vnet create -g $rg -n $vnet_name --address-prefix $vnet_prefix -l $location
az network vnet subnet create -g $rg -n $aks_subnet_name --vnet-name $vnet_name --address-prefix $aks_subnet_prefix
aks_subnet_id=$(az network vnet subnet show -n $aks_subnet_name --vnet-name $vnet_name -g $rg --query id -o tsv)
# Get latest supported/preview version
k8s_versions=$(az aks get-versions -l $location -o json)
if [[ "$preview_version" == "yes" ]]
then
k8s_version=$(echo $k8s_versions | jq '.orchestrators[]' | jq -rsc 'sort_by(.orchestratorVersion) | reverse[0] | .orchestratorVersion')
echo "Latest supported k8s version in $rg_location is $k8s_version (in preview)"
else
k8s_version=$(echo $k8s_versions | jq '.orchestrators[] | select(.isPreview == null)' | jq -rsc 'sort_by(.orchestratorVersion) | reverse[0] | .orchestratorVersion')
echo "Latest supported k8s version (not in preview) in $rg_location is $k8s_version"
fi
#####################
# With existing SPs #
#####################
# Get SP from AKV
keyvault_name=joseakv-airs
purpose=aks
keyvault_appid_secret_name=$purpose-sp-appid
keyvault_password_secret_name=$purpose-sp-secret
sp_app_id=$(az keyvault secret show --vault-name $keyvault_name -n $keyvault_appid_secret_name --query 'value' -o tsv)
sp_app_secret=$(az keyvault secret show --vault-name $keyvault_name -n $keyvault_password_secret_name --query 'value' -o tsv)
# Assign contributor role to the vnet
vnet_id=$(az network vnet show -n $vnet_name -g $rg --query id -o tsv)
az role assignment create --scope $vnet_id --assignee $sp_app_id --role Contributor
# Create AKS
az aks create -g $rg -n $aks_name -l $location \
-c 1 -s $vm_size -k $k8s_version --generate-ssh-keys \
--service-principal $sp_app_id --client-secret $sp_app_secret --skip-subnet-role-assignment \
--network-plugin azure --vnet-subnet-id $aks_subnet_id --service-cidr $aks_service_cidr \
--network-policy calico --load-balancer-sku Standard \
--node-resource-group "$aks_name"-iaas-"$RANDOM" \
--no-wait
# Other options you can use in the previous command
# --enable-private-cluster \
# --dns-name-prefix cloudtrooper \
########################
# Without existing SPs #
########################
# az aks create -g $rg -n $aks_name -l $location \
# -c 1 -s $vm_size -k $k8s_version --generate-ssh-keys \
# --network-plugin azure --vnet-subnet-id $aks_subnet_id --service-cidr $aks_service_cidr \
# --network-policy calico --load-balancer-sku Standard \
# --node-resource-group "$aks_name"-iaas-"$RANDOM" \
# --no-wait
###############
# App Gateway #
###############
# Only if not creating the app gw with the enable-addons command later on
az network vnet subnet create -n $appgw_subnet_name -g $rg --address-prefix $appgw_subnet_prefix --vnet-name $vnet_name
az network public-ip create -g $rg -n $appgw_pip_name --sku Standard
az network application-gateway create -g $rg -n $appgw_name --capacity 2 --sku $appgw_sku \
--frontend-port 80 --routing-rule-type basic \
--http-settings-port 80 --http-settings-protocol Http \
--public-ip-address $appgw_pip_name --vnet-name $vnet_name --subnet $appgw_subnet_name \
--no-wait
appgw_id=$(az network application-gateway show -n $appgw_name -g $rg --query id -o tsv)
wait_until_finished $appgw_id
########
# Wait #
########
aks_id=$(az aks show -n $aks_name -g $rg --query id -o tsv)
wait_until_finished $aks_id
# Get credentials for kubectl
az aks list -o table
az aks get-credentials -n $aks_name -g $rg --overwrite
kubectl get nodes
#########################
# Connect AKS and AppGW #
#########################
# OPTION 1: Using helm
# Extend RBAC role assignment
rg_id=$(az group show -n $rg --query id -o tsv)
az role assignment create --scope $rg_id --assignee $sp_app_id --role Contributor
# See https://azure.github.io/application-gateway-kubernetes-ingress/setup/install-existing/#using-a-service-principal
helm_file=/tmp/helm-config.yaml
helm repo add application-gateway-kubernetes-ingress https://appgwingress.blob.core.windows.net/ingress-azure-helm-package/
helm repo update
# wget https://raw.githubusercontent.com/Azure/application-gateway-kubernetes-ingress/master/docs/examples/sample-helm-config.yaml -O $helm_file
subscription_id=$(az account show --query id -o tsv)
tenant_id=$(az account show --query tenantId -o tsv)
sdk_auth="{
\"clientId\": \"$sp_app_id\",
\"clientSecret\": \"$sp_app_secret\",
\"subscriptionId\": \"$tenant_id\",
\"tenantId\": \"$tenant_id\",
\"activeDirectoryEndpointUrl\": \"https://login.microsoftonline.com\",
\"resourceManagerEndpointUrl\": \"https://management.azure.com/\",
\"activeDirectoryGraphResourceId\": \"https://graph.windows.net/\",
\"sqlManagementEndpointUrl\": \"https://management.core.windows.net:8443/\",
\"galleryEndpointUrl\": \"https://gallery.azure.com/\",
\"managementEndpointUrl\": \"https://management.core.windows.net/\"
}"
sdk_auth=$(echo $sdk_auth | base64 -w0)
cat <<EOF > $helm_file
verbosityLevel: 3
appgw:
subscriptionId: $subscription_id
resourceGroup: $rg
name: $appgw_name
usePrivateIP: false
shared: false
# kubernetes:
# watchNamespace: default
armAuth:
type: servicePrincipal
secretJSON: $sdk_auth
rbac:
enabled: true
EOF
helm install ingress-azure -f $helm_file application-gateway-kubernetes-ingress/ingress-azure
# helm install ingress-azure -f $helm_file application-gateway-kubernetes-ingress/ingress-azure --version 1.2.0-rc3
# OPTION 2: Using add-on, use an existing gateway
# appgw_subnet_id=$(az network vnet subnet show -n $appgw_subnet_name --vnet-name $vnet_name -g $rg --query id -o tsv)
# az aks enable-addons -n $aks_name -g $rg -a ingress-appgw --appgw-id $appgw_id --appgw-watch-namespace default
# OPTION 3: Using add-on, create gateway
# rg_id=$(az group show -n $rg --query id -o tsv)
# az role assignment create --scope $rg_id --assignee $sp_app_id --role Contributor
# az aks enable-addons -n $aks_name -g $rg -a ingress-appgw --appgw-subnet-prefix $appgw_subnet_prefix --appgw-name $appgw_name --appgw-watch-namespace default
#################
# Disconnect #
#################
# az aks disable-addons -a ingress-appgw -n $aks_name -g $rg
# helm uninstall ingress-azure
###################
# Deploy test app #
###################
# appgw_pip=$(az network public-ip show -n aksappgw-appgwpip -g $node_rg --query ipAddress -o tsv) # the PIP name will depend on how the AppGW was created
appgw_pip=$(az network public-ip show -n $appgw_pip_name -g $rg --query ipAddress -o tsv)
app_name=kuard02
image='gcr.io/kuar-demo/kuard-amd64:blue'
app_url=${app_name}.${appgw_pip}.nip.io
cat <<EOF | kubectl -n default apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
name: $app_name
labels:
app: $app_name
spec:
replicas: 1
selector:
matchLabels:
app: $app_name
template:
metadata:
labels:
app: $app_name
spec:
containers:
- name: $app_name
image: $image
ports:
- containerPort: 8080
---
apiVersion: v1
kind: Service
metadata:
name: $app_name
spec:
selector:
app: $app_name
ports:
- protocol: TCP
port: 80
targetPort: 8080
---
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: $app_name
annotations:
kubernetes.io/ingress.class: azure/application-gateway
spec:
rules:
- host: "$app_url"
http:
paths:
- path: /
backend:
serviceName: $app_name
servicePort: 80
EOF
echo "Access your app over http://${app_url}"
#########
# Tests #
#########
# Scale up/down deployments
k -n default scale deploy/kuard01 --replicas=0
k -n default get deploy
k -n default scale deploy/kuard01 --replicas=1
k -n default get deploy
###############
# Diagnostics #
###############
# appgw_rg=$(az aks show -n $aks_name -g $rg --query nodeResourceGroup -o tsv)
appgw_rg=$rg
az network vnet subnet list -g $rg -o table --vnet-name $vnet_name
az network application-gateway list -g $appgw_rg -o table
# appgw_name=$(az network application-gateway list -g $node_rg --query '[0].name' -o tsv)
az network application-gateway http-listener list -g $appgw_rg --gateway-name $appgw_name -o table
az network application-gateway frontend-ip list -g $appgw_rg --gateway-name $appgw_name -o table
az network application-gateway probe list -g $appgw_rg --gateway-name $appgw_name -o table
az network application-gateway address-pool list -g $appgw_rg --gateway-name $appgw_name -o table
az network application-gateway rule list -g $appgw_rg --gateway-name $appgw_name -o table
az network application-gateway rule show -g $appgw_rg --gateway-name $appgw_name -n rule1
az network application-gateway rule list -g $appgw_rg --gateway-name $appgw_name -o table
rule=$(az network application-gateway rule list -g $appgw_rg --gateway-name $appgw_name --query '[0].name' -o tsv)
az network application-gateway rule show -g $appgw_rg --gateway-name $appgw_name -n $rule
az network application-gateway url-path-map list -g $appgw_rg --gateway-name $appgw_name -o table
az network application-gateway http-settings list -g $appgw_rg --gateway-name $appgw_name -o table
az network application-gateway show-backend-health -g $appgw_rg -n $appgw_name --query 'backendAddressPools[].backendHttpSettingsCollection[].[backendHttpSettings.id,servers[].health]'
###########
# Cleanup #
###########
az group delete -n $rg -y --no-wait