Endpoint basic auth test (kubeflow#4447)

* add new method

* get kflogin

* updates

* return boolean

* link to e2e workflow

* updates

* enable test_endpoint in basic auth

* merge functions

* update func call

* break out syntax sugar

* add basic_auth_login_is_ready init

* finish basic auth test

* pass login info

* update test

prow_config.yaml

@@ -35,6 +35,7 @@ workflows:
       - testing/*
     kwargs:
       use_basic_auth: true
+      test_endpoint: true
       config_path: https://raw.githubusercontent.com/kubeflow/manifests/master/kfdef/kfctl_gcp_basic_auth.yaml
   # E2E tests for kfctl_existing_arrikto 
   - app_dir: kubeflow/kubeflow/testing/workflows

py/kubeflow/kubeflow/ci/kfctl_e2e_workflow.py

@@ -85,8 +85,7 @@ def __init__(self, name=None, namespace=None,
       namespace: Namespace for the workflow.
       config_path: Path to the KFDef spec file.
       bucket: The bucket to upload artifacts to. If not set use default determined by prow_artifacts.py.
-      test_endpoint: Whether to test the endpoint is ready. Should only
-        be true for IAP.
+      test_endpoint: Whether to test the endpoint is ready.
       use_basic_auth: Whether to use basic_auth.
       test_target_name: (Optional) Name to use as the test target to group
         tests.
@@ -555,6 +554,7 @@ def build(self):
                  "-o", "junit_suite_name=test_endpoint_is_ready_" + self.config_name,
                  "--app_path=" + self.app_dir,
                  "--app_name=" + self.app_name,
+                 "--use_basic_auth={0}".format(self.use_basic_auth),
               ]
 
       dependencies = [build_kfctl["name"]]

py/kubeflow/kubeflow/ci/kfctl_go_test_utils.py

@@ -1,5 +1,6 @@
 """Common reusable steps for kfctl go testing."""
 import datetime
+import json
 import logging
 import os
 import tempfile
@@ -99,6 +100,12 @@ def set_env_init_args(use_basic_auth, use_istio):
     # logging.info("Setting environment variables KUBEFLOW_USERNAME and KUBEFLOW_PASSWORD")
     os.environ["KUBEFLOW_USERNAME"] = "kf-test-user"
     os.environ["KUBEFLOW_PASSWORD"] = str(uuid.uuid4().hex)
+    with open(os.path.join(app_path, "login.json"), "w") as f:
+      login = {
+          "KUBEFLOW_USERNAME": os.environ["KUBEFLOW_USERNAME"],
+          "KUBEFLOW_PASSWORD": os.environ["KUBEFLOW_PASSWORD"],
+      }
+      json.dump(login, f)
     init_args = ["--use_basic_auth"]
   else:
     # Owned by project kubeflow-ci-deployment.

testing/gcp_util.py

@@ -26,6 +26,7 @@
 
 IAM_SCOPE = "https://www.googleapis.com/auth/iam"
 OAUTH_TOKEN_URI = "https://www.googleapis.com/oauth2/v4/token"
+COOKIE_NAME = "KUBEFLOW-AUTH-KEY"
 
 def get_service_account_credentials(client_id_key):
   # Figure out what environment we're running in and get some preliminary
@@ -75,7 +76,7 @@ def may_get_env_var(name):
   else:
     raise Exception("%s not set" % name)
 
-def endpoint_is_ready(url, wait_min=15):
+def iap_is_ready(url, wait_min=15):
   """
   Checks if the kubeflow endpoint is ready.
 
@@ -84,8 +85,9 @@ def endpoint_is_ready(url, wait_min=15):
   Returns:
     True if the url is ready
   """
-  service_account_credentials = get_service_account_credentials("CLIENT_ID")
+  google_open_id_connect_token = None
 
+  service_account_credentials = get_service_account_credentials("CLIENT_ID")
   google_open_id_connect_token = get_google_open_id_connect_token(
       service_account_credentials)
   # Wait up to 30 minutes for IAP access test.
@@ -97,6 +99,7 @@ def endpoint_is_ready(url, wait_min=15):
     num_req += 1
     logging.info("Trying url: %s", url)
     try:
+      resp = None
       resp = requests.request(
           "GET",
           url,
@@ -107,12 +110,65 @@ def endpoint_is_ready(url, wait_min=15):
           verify=False)
       logging.info(resp.text)
       if resp.status_code == 200:
-        logging.info("IAP is ready for %s!", url)
+        logging.info("Endpoint is ready for %s!", url)
         return True
       else:
         logging.info(
-            "%s: IAP not ready, request number: %s" % (url, num_req))
+            "%s: Endpoint not ready, request number: %s" % (url, num_req))
     except Exception as e:
-      logging.info("%s: IAP not ready, exception caught %s, request number: %s" %
+      logging.info("%s: Endpoint not ready, exception caught %s, request number: %s" %
                    (url, str(e), num_req))
   return False
+
+def basic_auth_is_ready(url, username, password, wait_min=15):
+  get_url = url + "/kflogin"
+  post_url = url + "/apikflogin"
+
+  req_num = 0
+  end_time = datetime.datetime.now() + datetime.timedelta(
+      minutes=wait_min)
+  while datetime.datetime.now() < end_time:
+    sleep(2)
+    req_num += 1
+    logging.info("Trying url: %s", get_url)
+    resp = requests.request(
+        "GET",
+        get_url,
+        verify=False)
+    if resp.status_code != 200:
+      logging.info("Basic auth login is not ready, request number %s: %s" % (req_num, get_url))
+      continue
+
+    logging.info("%s: endpoint is ready, testing login API; request number %s" % (get_url, req_num))
+    resp = requests.post(
+        post_url,
+        auth=(username, password),
+        headers={
+            "x-from-login": "true",
+        },
+        verify=False)
+    logging.info("%s: %s" % (post_url, resp.text))
+    if resp.status_code != 205:
+      logging.error("%s: login is failed", post_url)
+      return False
+
+    cookie = None
+    for c in resp.cookies:
+      if c.name == COOKIE_NAME:
+        cookie = c
+        break
+    if cookie is None:
+      logging.error("%s: auth cookie cannot be found; name: %s" % (post_url, COOKIE_NAME))
+      return False
+
+    resp = requests.get(
+        url,
+        cookies={
+            cookie.name: cookie.value,
+        },
+        verify=False)
+    logging.info("%s: %s" % (url, resp.status_code))
+    logging.info(resp.content)
+    return resp.status_code == 200
+
+  return False

testing/kfctl/endpoint_ready_test.py

@@ -1,4 +1,5 @@
 import datetime
+import json
 import logging
 import os
 import subprocess
@@ -19,7 +20,7 @@
 # We shouldn't need it to feel confident that kfctl is working.
 @pytest.mark.skipif(os.getenv("JOB_TYPE") == "presubmit",
                     reason="test endpoint doesn't run in presubmits")
-def test_endpoint_is_ready(record_xml_attribute, project, app_name):
+def test_endpoint_is_ready(record_xml_attribute, project, app_path, app_name, use_basic_auth):
   """Test that Kubeflow was successfully deployed.
 
   Args:
@@ -28,12 +29,20 @@ def test_endpoint_is_ready(record_xml_attribute, project, app_name):
   """
   util.set_pytest_junit(record_xml_attribute, "test_endpoint_is_ready")
 
-  # Owned by project kubeflow-ci-deployment.
-  os.environ["CLIENT_ID"] = "29647740582-7meo6c7a9a76jvg54j0g2lv8lrsb4l8g.apps.googleusercontent.com"
-  if not gcp_util.endpoint_is_ready(
-      "https://{}.endpoints.{}.cloud.goog".format(app_name, project),
-      wait_min=25):
-    raise Exception("Endpoint not ready")
+  url = "https://{}.endpoints.{}.cloud.goog".format(app_name, project)
+  if use_basic_auth:
+    with open(os.path.join(app_path, "login.json"), "r") as f:
+      login = json.load(f)
+      # Let it fail if login info cannot be found.
+      username = login["KUBEFLOW_USERNAME"]
+      password = login["KUBEFLOW_PASSWORD"]
+    if not gcp_util.basic_auth_is_ready(url, username, password):
+      raise Exception("Basic auth endpoint is not ready")
+  else:
+    # Owned by project kubeflow-ci-deployment.
+    os.environ["CLIENT_ID"] = "29647740582-7meo6c7a9a76jvg54j0g2lv8lrsb4l8g.apps.googleusercontent.com"
+    if not gcp_util.iap_is_ready(url):
+      raise Exception("IAP endpoint is not ready")
 
 if __name__ == "__main__":
   logging.basicConfig(level=logging.INFO,