failover: Make failover work over IP families

Originally the option ipv4_first and ipv6_first was taken into account
when resolving IP address.

When both families are resolvable but the primary is blocked on
firewall, the SSSD must switch to the socondary family.

src/providers/backend.h

@@ -56,6 +56,7 @@ struct be_svc_data {
 
     char *last_good_srv;
     int last_good_port;
+    int last_good_family;
     time_t last_status_change;
     bool run_callbacks;
 

src/providers/data_provider_fo.c

@@ -22,6 +22,7 @@
 #include <netdb.h>
 #include <arpa/inet.h>
 #include "providers/backend.h"
+#include "providers/fail_over.h"
 #include "resolv/async_resolv.h"
 
 struct be_svc_callback {
@@ -670,9 +671,13 @@ errno_t be_resolve_server_process(struct tevent_req *subreq,
         DEBUG(SSSDBG_TRACE_LIBS, "Saving the first resolved server\n");
         state->svc->first_resolved = state->srv;
     } else if (state->svc->first_resolved == state->srv) {
-        DEBUG(SSSDBG_OP_FAILURE,
-              "The fail over cycled through all available servers\n");
-        return ENOENT;
+        /* check that this is not the first attempt to second IP family */
+        if (fo_get_server_family(state->srv) !=
+            fo_get_server_secondary_family(state->srv)) {
+            DEBUG(SSSDBG_OP_FAILURE,
+                  "The fail over cycled through all available servers\n");
+            return ENOENT;
+        }
     }
 
     if (fo_get_server_name(state->srv)) {
@@ -713,6 +718,7 @@ errno_t be_resolve_server_process(struct tevent_req *subreq,
     if (state->svc->last_good_srv == NULL ||
         strcmp(fo_get_server_name(state->srv), state->svc->last_good_srv) != 0 ||
         fo_get_server_port(state->srv) != state->svc->last_good_port ||
+        fo_get_server_family(state->srv) != state->svc->last_good_family ||
         state->svc->run_callbacks ||
         srv_status_change > state->svc->last_status_change) {
         state->svc->last_status_change = srv_status_change;
@@ -727,6 +733,7 @@ errno_t be_resolve_server_process(struct tevent_req *subreq,
         talloc_free(state->svc->last_good_srv);
         state->svc->last_good_srv = srvname;
         state->svc->last_good_port = fo_get_server_port(state->srv);
+        state->svc->last_good_family = fo_get_server_family(state->srv);
 
         DLIST_FOR_EACH(callback, state->svc->callbacks) {
             callback->fn(callback->private_data, state->srv);

src/providers/fail_over.c

@@ -23,6 +23,7 @@
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+#include <sys/socket.h>
 #include <sys/time.h>
 
 #include <errno.h>
@@ -268,6 +269,8 @@ str_server_status(enum server_status status)
         return "working";
     case SERVER_NOT_WORKING:
         return "not working";
+    case SERVER_SECOND_FAMILY:
+        return "second family";
     }
 
     return "unknown server status";
@@ -481,6 +484,53 @@ service_destructor(struct fo_service *service)
     return 0;
 }
 
+static bool
+try_another_family(struct fo_server *server) {
+    enum restrict_family family_order;
+
+    if (
+        server == NULL ||
+        server->common == NULL ||
+        server->common->rhostent == NULL ||
+        server->service == NULL ||
+        server->service->ctx == NULL ||
+        server->service->ctx->opts == NULL) {
+        return false;
+    }
+
+    family_order = server->service->ctx->opts->family_order;
+
+    if (server->common->rhostent->family == AF_INET && family_order == IPV4_FIRST) {
+        return true;
+    }
+
+    if (server->common->rhostent->family == AF_INET6 && family_order == IPV6_FIRST) {
+        return true;
+    }
+
+    return false;
+}
+
+int
+fo_get_server_secondary_family(struct fo_server *server) {
+    if (
+        server == NULL ||
+        server->service == NULL ||
+        server->service->ctx == NULL ||
+        server->service->ctx->opts == NULL) {
+        return 0;
+    }
+
+    switch (server->service->ctx->opts->family_order) {
+    case IPV4_FIRST:
+        return AF_INET6;
+    case IPV6_FIRST:
+        return AF_INET;
+    default:
+        return 0;
+    }
+}
+
 int
 fo_new_service(struct fo_ctx *ctx, const char *name,
                datacmp_fn user_data_cmp,
@@ -1193,13 +1243,26 @@ fo_resolve_service_server(struct tevent_req *req)
                                         struct resolve_service_state);
     struct tevent_req *subreq;
     int ret;
+    enum restrict_family family_restriction;
 
+    family_restriction = state->fo_ctx->opts->family_order;
     switch (get_server_status(state->server)) {
+    case SERVER_SECOND_FAMILY:
+        switch (family_restriction) {
+        case IPV4_FIRST:
+            family_restriction = IPV6_ONLY;
+            break;
+        case IPV6_FIRST:
+            family_restriction = IPV4_ONLY;
+            break;
+        default:
+            break;
+        }
     case SERVER_NAME_NOT_RESOLVED: /* Request name resolution. */
         subreq = resolv_gethostbyname_send(state->server->common,
                                            state->ev, state->resolv,
                                            state->server->common->name,
-                                           state->fo_ctx->opts->family_order,
+                                           family_restriction,
                                            default_host_dbs);
         if (subreq == NULL) {
             tevent_req_error(req, ENOMEM);
@@ -1603,7 +1666,11 @@ fo_set_port_status(struct fo_server *server, enum port_status status)
     }
 
     if (!server->common || !server->common->name) return;
-
+    if (status == PORT_NOT_WORKING && try_another_family(server)) {
+        set_server_common_status(server->common, SERVER_SECOND_FAMILY);
+        server->port_status = PORT_NEUTRAL;
+        status = PORT_NEUTRAL;
+    }
     /* It is possible to introduce duplicates when expanding SRV results
      * into fo_server structures. Find the duplicates and set the same
      * status */
@@ -1657,6 +1724,17 @@ fo_get_server_port(struct fo_server *server)
     return server->port;
 }
 
+int
+fo_get_server_family(struct fo_server *server)
+{
+    if (server != NULL &&
+        server->common != NULL &&
+        server->common->rhostent) {
+        return server->common->rhostent->family;
+    }
+    return 0;
+}
+
 const char *
 fo_get_server_name(struct fo_server *server)
 {

src/providers/fail_over.h

@@ -49,6 +49,7 @@ enum server_status {
     SERVER_NAME_NOT_RESOLVED, /* We didn't yet resolved the host name. */
     SERVER_RESOLVING_NAME,    /* Name resolving is in progress. */
     SERVER_NAME_RESOLVED,     /* We resolved the host name but didn't try to connect. */
+    SERVER_SECOND_FAMILY,    /* We should try second protocol */
     SERVER_WORKING,           /* We successfully connected to the server. */
     SERVER_NOT_WORKING        /* We tried and failed to connect to the server. */
 };
@@ -198,6 +199,21 @@ void *fo_get_server_user_data(struct fo_server *server);
 
 int fo_get_server_port(struct fo_server *server);
 
+/*
+ * Get curently used/resolved inet family.
+ * Function returns AF_INET, AF_INET6 or 0 in case that
+ * name is not resolved yet.
+ */
+int fo_get_server_family(struct fo_server *server);
+
+/*
+ * Get secondary inet family if exists.
+ * Function returns AF_INET, AF_INET6 or 0 in case that there is no
+ * secondary family (for example if IPV4_ONLY is set). Note that
+ * this function returns what is configured, not what is actually used.
+ */
+int fo_get_server_secondary_family(struct fo_server *server);
+
 const char *fo_get_server_name(struct fo_server *server);
 
 const char *fo_get_server_str_name(struct fo_server *server);

src/providers/fail_over_srv.c

@@ -440,6 +440,7 @@ struct fo_resolve_srv_dns_ctx {
     char *hostname;
     char *sssd_domain;
     char *detected_domain;
+    bool last_family_tried;
 };
 
 struct fo_resolve_srv_dns_state {

src/tests/fail_over-tests.c

@@ -275,6 +275,8 @@ START_TEST(test_fo_resolve_service)
     get_request(ctx, service[0], EOK, 20, PORT_WORKING, -1);
     get_request(ctx, service[0], EOK, 20, PORT_WORKING, SERVER_WORKING);
     get_request(ctx, service[0], EOK, 20, -1, SERVER_NOT_WORKING);
+    get_request(ctx, service[0], EOK, 20, PORT_WORKING, SERVER_WORKING);
+    get_request(ctx, service[0], EOK, 20, -1, SERVER_NOT_WORKING);
     get_request(ctx, service[0], EOK, 80, PORT_WORKING, -1);
     get_request(ctx, service[0], EOK, 80, PORT_NOT_WORKING, -1);
     get_request(ctx, service[0], ENOENT, 0, -1, -1);