napalm - supported getters list

[writememe]

Hi NAPALM maintainers,

It would be great if there was some central functionality which provided NAPALM consumers a dictionary of the table shown in the getters support documentation:

Below is my attempt of using BeautifulSoup to web scrape this page and dynamically generate a dictionary of lists:
NOTE: You would have to sort out the Logger variable for logging, but hopefully you get the point

"""
A script to retrieve the HTML table
from the NAPALM supported getters
table

URL: https://napalm.readthedocs.io/en/latest/support/index.html#getters-support-matrix
"""
# Import modules
import pandas as pd
import emoji
import requests
from bs4 import BeautifulSoup
import os
import pathlib
import time
import json
import urllib3

# Disable URL warnings
urllib3.disable_warnings()


# Get path of the current directory under which the file is executed.
dirname = os.path.dirname(__file__)


# NOTE: This is a placeholder for initialising the logger module
# through whatever means is used to do that.
# You can remove logging from the final solution.
Logger = None

# Create a log directory and/or ensure that it exists
log_dir = "logs"
pathlib.Path(log_dir).mkdir(exist_ok=True)
# Set the logging level. Valid options: DEBUG, INFO, WARNING, CRITICAL
level = "INFO"
# Add timestamp to log for troubleshooting
timestamp = time.strftime("%Y-%m-%d-%H-%M-%S")
# Assign log name
log_name = f"{log_dir}/napalm_getter_matrix-{timestamp}.log"
# Initialise the logger
logger = Logger(log_level=level, log_name=log_name, app_name="napalm_getter_matrix")


def get_napalm_getters_table_to_df(
    url="https://napalm.readthedocs.io/en/latest/support/index.html#getters-support-matrix",
):
    """
    This function retrieves the NAPALM supported getter matrix table
    from the NAPALM official website and converts it to a Pandas
    dataframe for further processing by other functions.

    :param url: The NAPALM URL where the table is stored
        Default: https://napalm.readthedocs.io/en/latest/support/index.html#getters-support-matrix
    :type url: string

    :return df: The NAPALM support table in a pandas dataframe format.
    :type df: dataframe
    """
    # Get the NAPALM page
    page = requests.get(url, verify=False)  # nosec (ssl ignore)
    # Parse the page using BeautifulSoup
    soup = BeautifulSoup(page.text, "html.parser")
    # Find the getters support matrix section of the webpage
    # and assign to a variable
    getter_support_section = soup.find(
        "div", {"class": "section", "id": "getters-support-matrix"}
    )
    # Find the table inside this section
    # NOTE: At time of writing, there is only one table
    getter_matrix_table = getter_support_section.find("table")
    # Debug printout
    logger.debug(getter_matrix_table)
    # Instantiate an empty list of headers to add to
    headers = []
    for table_header in getter_matrix_table.find_all("th"):
        title = table_header.text
        headers.append(title)
    # Debug printout
    logger.debug(f"Headers: {headers}")
    # Initalise a pandas dataframe with the columns as the headers
    df = pd.DataFrame(columns=headers)
    # Iterate over the table rows, but skip the first row as it's
    # the header row
    for table_row in getter_matrix_table.find_all("tr")[1:]:
        # Find all the table data
        data = table_row.find_all("td")
        # Strip all the text out of the table data and assign to a variable
        row_data = [td.text.strip() for td in data]
        # Gather the length of the dataframe
        length = len(df)
        # Add the row data back into the dataframe
        df.loc[length] = row_data
        # We need to rename the original header, so we can access the getter_name
    df = df.rename(columns={"": "getter_name"})
    # Debug printout
    logger.debug(df)
    # Return the dataframe
    return df


def parse_df_to_support_matrix_dict(df):
    """
    Parse the NAPALM support matrix dataframe into a dictionary
    of results which contained all the supported getters for each platform.

    :params df: The NAPALM support table in a pandas dataframe format.
    :type df: dataframe

    :return getter_results: A dictionary which contains the supported getters for
    each platform
        Example:
            {
                'eos': ['get_bgp_config', 'get_facts', 'is_alive'],
                'ios': ['get_bgp_config', 'get_facts', 'is_alive'],
                'iosxr': ['get_bgp_config', 'get_facts', 'is_alive'],
                'junos': ['get_mac_address_table', 'get_facts', 'is_alive'],
                'nxos': ['get_network_instances', 'get_facts', 'is_alive'],
                'nxos_ssh': ['get_bgp_config', 'get_facts', 'is_alive'],
            }
    :type getter_results: dict
    """
    # Instantiate empty lists to populate
    # getter results into
    (
        eos_getters,
        ios_getters,
        iosxr_getters,
        junos_getters,
        nxos_getters,
        nxos_ssh_getters,
    ) = ([], [], [], [], [], [])
    # Iterate over pandas dataframe
    for row in df.itertuples():
        # Take the emoji value, and convert it to a string, so we
        # can perform a string comparison match. NOTE: ':check_mark_button:'
        # indicates that it is supported.
        # EOS if block
        if emoji.demojize(row.EOS) == ":check_mark_button:":
            # Debug printout
            logger.debug(f"Match found {row.getter_name}")
            # Add to EOS list
            eos_getters.append(str(row.getter_name))
        # IOS if block
        if emoji.demojize(row.IOS) == ":check_mark_button:":
            # Debug printout
            logger.debug(f"Match found {row.getter_name}")
            # Add to IOS list
            ios_getters.append(str(row.getter_name))
        # IOSXR if block
        if emoji.demojize(row.IOSXR) == ":check_mark_button:":
            # Debug printout
            logger.debug(f"Match found {row.getter_name}")
            # Add to IOSXR list
            iosxr_getters.append(str(row.getter_name))
        # JUNOS if block
        if emoji.demojize(row.JUNOS) == ":check_mark_button:":
            # Debug printout
            logger.debug(f"Match found {row.getter_name}")
            # Add to JUNOS list
            junos_getters.append(str(row.getter_name))
        # NXOS if block
        if emoji.demojize(row.NXOS) == ":check_mark_button:":
            # Debug printout
            logger.debug(f"Match found {row.getter_name}")
            # Add to NXOS list
            nxos_getters.append(str(row.getter_name))
        # NXOS_SSH if block
        if emoji.demojize(row.NXOS_SSH) == ":check_mark_button:":
            # Debug printout
            logger.debug(f"Match found {row.getter_name}")
            # Add to NXOS_SSH list
            nxos_ssh_getters.append(str(row.getter_name))
    # Debug printouts
    logger.debug(f"eos_getters: {eos_getters}")
    logger.debug(f"ios_getters: {ios_getters}")
    logger.debug(f"iosxr_getters: {iosxr_getters}")
    logger.debug(f"junos_getters: {junos_getters}")
    logger.debug(f"nxos_getters: {nxos_getters}")
    logger.debug(f"nxos_ssh_getters: {nxos_ssh_getters}")
    # Instantiate results dict
    getter_results = dict()
    # Store the results of all the supported getters into the getter
    # results using the platform as the key
    getter_results["eos"] = eos_getters
    getter_results["ios"] = ios_getters
    getter_results["iosxr"] = iosxr_getters
    getter_results["junos"] = junos_getters
    getter_results["nxos"] = nxos_getters
    getter_results["nxos_ssh"] = nxos_ssh_getters
    # Informational printout
    logger.info(f"Getter Results:\n\n{getter_results}")
    # Result getter results dict
    return getter_results


def clean_getter_results(getter_results):
    """
    Take the getter results dictionary and modify appropriately by "cleaning"
    the dictionary results by performing the following:

    - Find instances of "get_<getter>" and replace with "<getter>"
    - Remove unwanted getters which don't store results. For example, "is_alive" and "ping"

    :param getter_results: A dictionary which contains the supported getters for
    each platform
        Example:
            {
                'eos': ['get_bgp_config', 'get_facts', 'is_alive'],
                'ios': ['get_bgp_config', 'get_facts', 'is_alive'],
                'iosxr': ['get_bgp_config', 'get_facts', 'is_alive'],
                'junos': ['get_mac_address_table', 'get_facts', 'is_alive'],
                'nxos': ['get_network_instances', 'get_facts', 'is_alive'],
                'nxos_ssh': ['get_bgp_config', 'get_facts', 'is_alive'],
            }
    :type getter_results: dict

    :return cleaned_getter_results: The 'getter_results' dictionary after it has been 'cleaned'
        Example using the above docstring:
        {
            'eos': ['bgp_config', 'facts'],
            'ios': ['bgp_config', 'facts'],
            'iosxr': ['bgp_config', 'facts'],
            'junos': ['mac_address_table', 'facts'],
            'nxos': ['network_instances', 'facts'],
            'nxos_ssh': ['bgp_config', 'facts'],
        }
    :type cleaned_getter_results: dict
    """
    cleaned_getter_results = dict()
    # Instantiate empty lists to populate
    # getter results into
    (
        eos_cleaned_getters,
        ios_cleaned_getters,
        iosxr_cleaned_getters,
        junos_cleaned_getters,
        nxos_cleaned_getters,
        nxos_ssh_cleaned_getters,
    ) = ([], [], [], [], [], [])
    # Specify a list of getters which aren't used frequently
    # for NAPALM getters to retrieve data.
    unwanted_getters = [
        "is_alive",
        "ping",
        "traceroute",
        "get_route_to",
    ]
    # Instantiate an empty list of platforms, to be used for further processing
    platform_list = []
    # Iterate over the keys in the getter_results (i.e. "ios", "eos" etc.)
    for platform in getter_results:
        # Append the platform name to the platform list
        platform_list.append(platform)
    # Debug printout
    logger.info(f"Platform list: {platform_list}")
    # Iterate the platforms in the platform list
    for platform in platform_list:
        # Iterate over the keys in the getter results and use the platform as the
        # key
        for getter in getter_results[platform]:
            # If the getter found, is in the unwanted list, discard the result
            # and perform printout
            if getter in unwanted_getters:
                # Debug printout
                logger.debug(f"Unwanted getter: {getter}, discarding")
            # Else, clean the getter
            else:
                # Replace "get_<getter>" with "<getter>" and assign to a variable
                # For example, 'get_facts' with 'facts'
                cleaned_getter = getter.replace("get_", "")
                # Debug print
                logger.debug(f"Processed getter: {cleaned_getter}")
                # Add the cleaned getter to the appropriate list at the top of the
                # function, using eval to dynamically elect the correct list
                eval(f"{platform}_cleaned_getters").append(  # nosec (eval exemption)
                    cleaned_getter
                )
    # Store the results of all the supported getters into the getter
    # results using the platform as the key
    cleaned_getter_results["eos"] = eos_cleaned_getters
    cleaned_getter_results["ios"] = ios_cleaned_getters
    cleaned_getter_results["iosxr"] = iosxr_cleaned_getters
    cleaned_getter_results["junos"] = junos_cleaned_getters
    cleaned_getter_results["nxos"] = nxos_cleaned_getters
    cleaned_getter_results["nxos_ssh"] = nxos_ssh_cleaned_getters
    # Informational printout
    logger.info(f"Cleaned Getter Results:\n\n{cleaned_getter_results}")
    # Return cleaned getter results
    return cleaned_getter_results


def retrieve_napalm_support_matrix(offline=True, clean=True):
    """
    Main function to retrieve a dictionary of getters from
    the official NAPALM website and return for processing
    in other automation solutions

    :param clean: A boolean to present back a copy of the "cleaned" getter
    results. This is so you get a cleaned list which you can feed into
    nornir tasks.
        Default: True

    There are conditional returns returned based on the clean boolean.

    :return getter_results: A dictionary which contains the supported getters for
    each platform
        Example:
            {
                'eos': ['get_bgp_config', 'get_facts', 'is_alive'],
                'ios': ['get_bgp_config', 'get_facts', 'is_alive'],
                'iosxr': ['get_bgp_config', 'get_facts', 'is_alive'],
                'junos': ['get_mac_address_table', 'get_facts', 'is_alive'],
                'nxos': ['get_network_instances', 'get_facts', 'is_alive'],
                'nxos_ssh': ['get_bgp_config', 'get_facts', 'is_alive'],
            }
    :type getter_results: dict

    :return cleaned_getter_results: The 'getter_results' dictionary after it has been 'cleaned'
        Example using the above docstring:
        {
            'eos': ['bgp_config', 'facts'],
            'ios': ['bgp_config', 'facts'],
            'iosxr': ['bgp_config', 'facts'],
            'junos': ['mac_address_table', 'facts'],
            'nxos': ['network_instances', 'facts'],
            'nxos_ssh': ['bgp_config', 'facts'],
        }
    :type cleaned_getter_results: dict

    """
    # Take the NAPALM support URL and retrieve the NAPALM
    # getter table into a pandas dataframe
    getter_support_df = get_napalm_getters_table_to_df(
        url="https://napalm.readthedocs.io/en/latest/support/index.html"
    )
    # Parse the pandas dataframe into a dictionary of results
    getter_results = parse_df_to_support_matrix_dict(getter_support_df)
    # If clean is set to True, clean the results and return the cleaned
    # getter results
    if clean:
        cleaned_getter_results = clean_getter_results(getter_results)
        return cleaned_getter_results
    # Else, return the getter_results which are not cleaned
    else:
        return getter_results


def example_main():
    """
    An example main workflow.
    """
    # Setup variable to control the logic of the example
    clean = True
    # Retrieve the cleaned results
    cleaned_results = retrieve_napalm_support_matrix(clean=clean)
    # Prettify the response to a variable
    cleaned_results_pretty = json.dumps(cleaned_results, indent=2)
    # Printout the variable
    logger.debug(cleaned_results_pretty)
    # Example code of gathering the IOS getters by accessing 'ios' key
    ios_getters = cleaned_results["ios"]
    # Printout the ios getters results
    logger.info(f"IOS Getters: {ios_getters}")
    if clean:
        # If clean is set to True, setup different filename
        output_file = f"{dirname}/sample_napalm_cleaned_getters_results.json"
    else:
        output_file = f"{dirname}/sample_napalm_raw_getters_results.json"
    # Save the results to an example file
    with open(output_file, "w+") as cleaned_getters_json:
        json.dump(cleaned_results, cleaned_getters_json, indent=2)
        logger.info(f"File saved to {cleaned_getters_json.name}")


if __name__ == "__main__":
    example_main()

Below are the outputs:

sample_napalm_cleaned_getters_results.json

{
  "eos": [
    "bgp_config",
    "bgp_neighbors",
    "bgp_neighbors_detail",
    "config",
    "environment",
    "facts",
    "interfaces",
    "interfaces_counters",
    "interfaces_ip",
    "lldp_neighbors",
    "lldp_neighbors_detail",
    "mac_address_table",
    "network_instances",
    "ntp_servers",
    "ntp_stats",
    "optics",
    "snmp_information",
    "users",
    "vlans"
  ],
  "ios": [
    "bgp_config",
    "bgp_neighbors",
    "bgp_neighbors_detail",
    "config",
    "environment",
    "facts",
    "interfaces",
    "interfaces_counters",
    "interfaces_ip",
    "ipv6_neighbors_table",
    "lldp_neighbors",
    "lldp_neighbors_detail",
    "mac_address_table",
    "network_instances",
    "ntp_peers",
    "ntp_servers",
    "ntp_stats",
    "optics",
    "probes_config",
    "snmp_information",
    "users",
    "vlans"
  ],
  "iosxr": [
    "bgp_config",
    "bgp_neighbors",
    "bgp_neighbors_detail"
    "config",
    "environment",
    "facts",
    "interfaces",
    "interfaces_counters",
    "interfaces_ip",
    "lldp_neighbors",
    "lldp_neighbors_detail",
    "mac_address_table",
    "ntp_peers",
    "ntp_servers",
    "ntp_stats",
    "probes_config",
    "probes_results",
    "snmp_information",
    "users"
  ],
  "junos": [
    "bgp_config",
    "bgp_neighbors",
    "bgp_neighbors_detail",
    "config",
    "environment",
    "facts",
    "interfaces",
    "interfaces_counters",
    "interfaces_ip",
    "ipv6_neighbors_table",
    "lldp_neighbors",
    "lldp_neighbors_detail",
    "mac_address_table",
    "network_instances",
    "ntp_peers",
    "ntp_servers",
    "ntp_stats",
    "optics",
    "probes_config",
    "probes_results",
    "snmp_information",
    "users",
    "vlans"
  ],
  "nxos": [
    "bgp_neighbors",
    "config",
    "environment",
    "facts",
    "interfaces",
    "interfaces_ip",
    "lldp_neighbors",
    "lldp_neighbors_detail",
    "mac_address_table",
    "network_instances",
    "ntp_peers",
    "ntp_servers",
    "ntp_stats",
    "snmp_information",
    "users",
    "vlans"
  ],
  "nxos_ssh": [
    "bgp_neighbors",
    "config",
    "environment",
    "facts",
    "interfaces",
    "interfaces_counters",
    "interfaces_ip",
    "lldp_neighbors",
    "lldp_neighbors_detail",
    "mac_address_table",
    "ntp_peers",
    "ntp_servers",
    "optics",
    "snmp_information",
    "users",
    "vlans"
  ]
}

sample_napalm_raw_getters_results.json

{
  "eos": [
    "get_bgp_config",
    "get_bgp_neighbors",
    "get_bgp_neighbors_detail",
    "get_config",
    "get_environment",
    "get_facts",
    "get_interfaces",
    "get_interfaces_counters",
    "get_interfaces_ip",
    "get_lldp_neighbors",
    "get_lldp_neighbors_detail",
    "get_mac_address_table",
    "get_network_instances",
    "get_ntp_servers",
    "get_ntp_stats",
    "get_optics",
    "get_route_to",
    "get_snmp_information",
    "get_users",
    "get_vlans",
    "is_alive",
    "ping",
    "traceroute"
  ],
  "ios": [
    "get_bgp_config",
    "get_bgp_neighbors",
    "get_bgp_neighbors_detail",
    "get_config",
    "get_environment",
    "get_facts",
    "get_interfaces",
    "get_interfaces_counters",
    "get_interfaces_ip",
    "get_ipv6_neighbors_table",
    "get_lldp_neighbors",
    "get_lldp_neighbors_detail",
    "get_mac_address_table",
    "get_network_instances",
    "get_ntp_peers",
    "get_ntp_servers",
    "get_ntp_stats",
    "get_optics",
    "get_probes_config",
    "get_snmp_information",
    "get_users",
    "get_vlans",
    "is_alive",
    "ping",
    "traceroute"
  ],
  "iosxr": [
    "get_bgp_config",
    "get_bgp_neighbors",
    "get_bgp_neighbors_detail",
    "get_config",
    "get_environment",
    "get_facts",
    "get_interfaces",
    "get_interfaces_counters",
    "get_interfaces_ip",
    "get_lldp_neighbors",
    "get_lldp_neighbors_detail",
    "get_mac_address_table",
    "get_ntp_peers",
    "get_ntp_servers",
    "get_ntp_stats",
    "get_probes_config",
    "get_probes_results",
    "get_snmp_information",
    "get_users",
    "is_alive",
    "traceroute"
  ],
  "junos": [
    "get_bgp_config",
    "get_bgp_neighbors",
    "get_bgp_neighbors_detail",
    "get_config",
    "get_environment",
    "get_facts",
    "get_interfaces",
    "get_interfaces_counters",
    "get_interfaces_ip",
    "get_ipv6_neighbors_table",
    "get_lldp_neighbors",
    "get_lldp_neighbors_detail",
    "get_mac_address_table",
    "get_network_instances",
    "get_ntp_peers",
    "get_ntp_servers",
    "get_ntp_stats",
    "get_optics",
    "get_probes_config",
    "get_probes_results",
    "get_snmp_information",
    "get_users",
    "get_vlans",
    "is_alive",
    "ping",
    "traceroute"
  ],
  "nxos": [
    "get_bgp_neighbors",
    "get_config",
    "get_environment",
    "get_facts",
    "get_interfaces",
    "get_interfaces_ip",
    "get_lldp_neighbors",
    "get_lldp_neighbors_detail",
    "get_mac_address_table",
    "get_network_instances",
    "get_ntp_peers",
    "get_ntp_servers",
    "get_ntp_stats",
    "get_snmp_information",
    "get_users",
    "get_vlans",
    "is_alive",
    "ping",
    "traceroute"
  ],
  "nxos_ssh": [
    "get_bgp_neighbors",
    "get_config",
    "get_environment",
    "get_facts",
    "get_interfaces",
    "get_interfaces_counters",
    "get_interfaces_ip",
    "get_lldp_neighbors",
    "get_lldp_neighbors_detail",
    "get_mac_address_table",
    "get_ntp_peers",
    "get_ntp_servers",
    "get_optics",
    "get_snmp_information",
    "get_users",
    "get_vlans",
    "is_alive",
    "ping",
    "traceroute"
  ]
}

Anyway, it would be cool if there was internal functionality offered to retrieve these lists. Whilst the solution above works, it's obviously fragile to many factors such as the table formatting changing on the official docs website.

It would also allow tools like this to dynamically generate the static getter lists.

[Kircheneer]

I think the list is dynamically built here. You might be able to use that directly.

[writememe]

Thanks @Kircheneer, I had a look at that after you've pointed it out.

[ktbyers]

I commented in Slack, but I don't really see the need for this (open to discussions on it).

Currently you could just use try/except to handle (will get NotImpletedError if not supported).

Or you could copy/reuse that def build_getters_support_matrix if you really need a programmatic way to build that in advance (i.e. the try/except pattern wasn't acceptable).

[writememe]

Thanks @ktbyers . Happy with your response and as mentioned above, I already having a working solution to the problem I was looking to solve.