hoorex.in

#!/usr/bin/env python
#
# ex:set ai shiftwidth=4 inputtab=spaces smarttab noautotab:

"""
Copyright (c) 2017-18  Christoph Willing, Brisbane Australia

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program (see LICENSE.md). If not,
see <http://www.gnu.org/licenses/>.
"""

from __future__ import print_function

import sys, os
import fnmatch
import subprocess
import pickle
import argparse
import logging
import ConfigParser
import re
from string import printable


# How we were called
(apppath, appname) = os.path.split(sys.argv[0])

HOOREX_VERSION = 'X.Y.Z'

# XDG definitions
_home = os.path.expanduser('~')
xdg_data_home = os.environ.get('XDG_DATA_HOME') or os.path.join(_home, '.local', 'share')
xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or os.path.join(_home, '.config')

logging.basicConfig(format='%(levelname)s: %(message)s')
hlog = logging.getLogger(appname)

# The field name to search for in .info files
DEPTARGET = os.getenv('DEPTARGET','REQUIRES')

# How far into requirements tree to go
DEPTH_LIMIT  = 10


def main():
    # User configuration
    default_config = load_user_config()

    # Command line arguments
    parser = argparse.ArgumentParser(description="This program shows which \
                    other packages require a given SBo package")
    parser.add_argument("-f", "--force", action="store_true", dest="force",
                default=default_config.getboolean('HooRex', 'force'),
                help="force (re)processing of repo data")
    parser.add_argument("-m", "--multilevel", action="store_true", dest="multilevel",
                default=default_config.getboolean('HooRex', 'multilevel'),
                help="after finding immediate requirers, show who requires those")
    parser.add_argument("-d", "--debug", action="store_true", dest="debug_mode",
                default=default_config.getboolean('HooRex', 'debug'),
                help="show additional additional debugging information, \
                    not just calculated package names")
    parser.add_argument('-g', '--group', dest='sbo_group', action='store', nargs='+',
                help="select all packages in the given SBo category \
                    e.g. --group audio")
    parser.add_argument("-1", "--column", action="store_true", dest="output_column",
                default=default_config.getboolean('HooRex', 'output_column'),
                help="show output as a sinlge column")
    parser.add_argument("-l", "--long", action="store_true", dest="output_long",
                default=default_config.getboolean('HooRex', 'output_long'),
                help="show containing directory (category) for each package")
    parser.add_argument("-i", "--installed", action="store_true", dest="output_installed",
                default=default_config.getboolean('HooRex', 'output_installed'),
                help="show only packages already installed, except that target package(s) are included even if not already installed")
    parser.add_argument("-I", "--installed_strict", action="store_true", dest="output_strictly_installed",
                default=False,
                help="really show only packages already installed, not even target packages")
    parser.add_argument("-p", "--dataPath", action="store", dest="data_path",
                help="specify the directory path to be used for repo data cache storage")
    parser.add_argument("-r", "--reverse", action="store_true", dest="reverse_lookup",
                default=default_config.getboolean('HooRex', 'reverse_lookup'),
                help="show build dependencies for target package(s)")
    parser.add_argument("-R", "--restricted", action="store_true", dest="output_only_input",
                default=False,
                help="show only packages from the input list")
    parser.add_argument('-s', '--slackbuilds', dest='sbo_path', action='store',
                help="set the full filesystem path to the local slackbuilds \
                    repository e.g. -s /home/jerry/slackbuilds")
    parser.add_argument("-U", "--unknown_strict", action="store_true", dest="unknown_strict",
                default=default_config.getboolean('HooRex', 'unknown_strict'),
                help="Be strict about about unknown package names - report them, then exit")
    parser.add_argument("-V", "--version", dest="app_version", action="store_true",
                help="display version number and exit")
    parser.add_argument("target", metavar='PKG', type=str, nargs='*',
                help="package(s) to process")

    args = parser.parse_args()

    # Allow packages to be taken from stdin as well as a command argument
    if not sys.stdin.isatty():
        pipe_targets = sys.stdin.read().split()
    else:
        pipe_targets = []

    if args.app_version:
        print(HOOREX_VERSION)
        sys.exit(0)

    if args.debug_mode:
        hlog.setLevel(logging.DEBUG)
        hlog.debug("VERBOSE mode set at command line")
    else:
        hlog.setLevel(logging.INFO)

    if args.sbo_path is not None:
        sbo_path = args.sbo_path.split()[-1]
        # If no SBO_PATH exists, set it in config defaults
        if  not default_config.has_option('HooRex', 'sbo_path'):
            user_config_set(default_config, 'sbo_path', sbo_path)
    elif default_config.has_option('HooRex', 'sbo_path'):
        sbo_path = default_config.get('HooRex', 'sbo_path')
        hlog.debug("Set sbo_path to %s from config file" % sbo_path)
    else:
        sbo_path = find_sbo_path()
    hlog.debug("sbo_path: %s" % sbo_path)

    # Repo Data
    if args.data_path is not None:
        data_dir = args.data_path
    else:
        data_dir = os.path.join(xdg_data_home, appname)
    try:
        os.makedirs(data_dir)
    except:
        if not os.path.exists(data_dir):
            hlog.info("Couldn't create directory (%s) for the repo data" % data_dir)
            sys.exit(3)
        else:
            # No problem - this path already exists
            pass
    SAVED_DATA=os.path.join(data_dir, 'repoData.pkl')

    # Our personal database of package relationships
    PkgData = dict()

    if args.force == False and os.path.exists(SAVED_DATA):
        # First try to load preexisting data
        hlog.debug("Loading existing repo data")
        pkl_file = open(SAVED_DATA, 'rb')
        PkgData = pickle.load(pkl_file)
        pkl_file.close()
        hlog.debug("PkgData loaded from file OK")
    else:
        if args.force == True:
            hlog.debug("(re)build of repo data forced by -f option - please wait ...")
        elif not os.path.exists(SAVED_DATA):
            hlog.debug("Couldn't open data file, generating new data - please wait ...")
        build_dicts(sbo_path, PkgData, deptarget=DEPTARGET)
        output = open(SAVED_DATA, 'wb')
        # Pickle the list using the highest protocol available.
        pickle.dump(PkgData, output, -1)
        output.close()

    # At version 0.6.0, PkgData has an additional entry PkgDataInfo.
    # Force an update if current repo doesn't have it
    if not PkgData.has_key('PkgDataInfo'):
        hlog.debug("(re)build of repo data forced by out of date data format - please wait ...")
        build_dicts(sbo_path, PkgData, deptarget=DEPTARGET)
        output = open(SAVED_DATA, 'wb')
        # Pickle the list using the highest protocol available.
        pickle.dump(PkgData, output, -1)
        output.close()

    PkgDataInfo = PkgData['PkgDataInfo']
    hlog.debug("REPO has version %s    " % PkgDataInfo['hoorex_data_version'])
    hlog.debug("REPO uses deptarget %s " % PkgDataInfo['deptarget'])
    DirectRequires = PkgData['DirectRequires']
    PkgRequires = PkgData['PkgRequires']
    PkgNeededBy = PkgData['PkgNeededBy']
    PkgCategory = PkgData['PkgCategory']

    # Warn and exit if DEPTARGET doesn't match current data
    if DEPTARGET != PkgDataInfo['deptarget']:
        hlog.critical("Requested dependency field (%s) doesn't match current data (using %s)" % (DEPTARGET,PkgDataInfo['deptarget']))
        hlog.critical("Please regenerate data index with something like:")
        hlog.critical("\tDEPTARGET=%s hoorex -f" % DEPTARGET)
        sys.exit(0)


    # This is our input - the package names being queried
    # via pipe and/or command line or a predefined group.
    #
    # First handle -g|--group
    # Typically its a category of the SBo repo
    # but we can define special groups ourselves here like the 'all' group.
    if args.sbo_group:
        targets = []
        if 'all' in args.sbo_group:
            targets = PkgCategory.keys()
        for (k,v) in PkgCategory.iteritems():
            if v in args.sbo_group:
                targets.append(k)
    else:
        # We strip off any category directory name
        # that might be attached e.g. from the --long output of a previous stage
        # of a pipeline.
        targets = [os.path.split(t)[-1] for t in list(set(pipe_targets + args.target))]

    # This is "raw" output (unsorted, unfiltered)
    multi_list = []

    # First check that the input targets exist in the repo
    bad_target = False
    for target in reversed(targets):
        if not PkgRequires.has_key(target):
            bad_target = True
            if args.unknown_strict:
                hlog.critical("Unknown package: %s" %target)
            else:
                targets.remove(target)
    if bad_target and args.unknown_strict:
        sys.exit(2)


    hlog.debug("Processing initial targets of: %s" % targets)
    if args.reverse_lookup:
        hlog.debug("REVERSE mode set at command line")
        multi_list.append(targets)
        for package in targets:
            if PkgRequires.has_key(package):
                multi_list.append(PkgRequires[package])
    else:
        multilevel_depth = 0
        multi_list.append(targets)
        while True:
            if len(targets) < 1:
                break
            hlog.debug("At LEVEL %d, required by: %s" % (multilevel_depth, targets))
            pkglist = []
            for package in targets:
                if DirectRequires.has_key(package):
                    if PkgNeededBy.has_key(package):
                        hlog.debug("\t%s is needed by: %s" % (package, PkgNeededBy[package]))
                        pkglist.extend(PkgNeededBy[package])
                    else:
                        hlog.debug("\t%s isn't needed by any other package" % package)
                else:
                    hlog.debug("\tSkipping package \"%s\" (unknown package)" % package)

            if len(pkglist) > 0:
                multi_list.append(pkglist)
            if not args.multilevel or len(targets) < 1:
                break
            targets = list(set(pkglist))
            multilevel_depth += 1
            if multilevel_depth > DEPTH_LIMIT:
                break

    # Everything (including initial pkgs enquired about)
    hlog.debug(multi_list)

    # Flatten multi_list, then filter and sort for output
    raw_output = []
    for pkgs in multi_list:
        for prereq in pkgs:
            # Step 0 filter: only add it if we know about it
            if PkgRequires.has_key(prereq):
                raw_output.append(prereq)
    # Filter the list, if necessary
    sorted_output = []
    if args.output_only_input:
        # -R option given
        sorted_output = sort_output([name for name in targets if name in set(raw_output)], PkgRequires)
    elif args.output_installed or args.output_strictly_installed:
        repo_path = default_config.get('HooRex', 'slackware_repo')
        if args.output_strictly_installed:
            # -I option
            all_installed = ['-'.join(pkgname.split('-')[:-3]) for pkgname in os.listdir(repo_path)]
        else:
            # -i option
            all_installed = set(targets + ['-'.join(pkgname.split('-')[:-3]) for pkgname in os.listdir(repo_path)])
        sorted_output = sort_output([name for name in all_installed if name in set(raw_output)], PkgRequires)
    else:
        # No filter on output
        sorted_output = sort_output(list(set(raw_output)), PkgRequires)

    # OUTPUT
    for req in sorted_output:
        if args.output_long:
            if args.output_column:
                print(os.path.join(PkgCategory[req], req))
            else:
                print(os.path.join(PkgCategory[req], req), end=' ')
        else:
            if args.output_column:
                print(req)
            else:
                print(req, end=' ')

    if not args.output_column:
        print()


def build_dicts(sbo_path, PkgData, deptarget='REQUIRES'):

    here = os.getcwd()
    os.chdir(sbo_path)
    hlog.debug("Using %s to index dependencies" % deptarget)
    reg_requires = re.compile('(?P<name>REQUIRES)="(?P<value>.*?)"', re.DOTALL)
    if deptarget == 'PREREQS':
        reg = re.compile('(?P<name>PREREQS)="(?P<value>.*?)"', re.DOTALL)
    elif deptarget == 'REQUIRES':
        reg = reg_requires
    else:
        hlog.critical("Unknown dependency target name (%s)" % deptarget)
        return

    PkgDataInfo = dict()
    DirectRequires = dict()
    PkgRequires = dict()
    PkgCategory = dict()
    PkgNeededBy = dict()

    PkgDataInfo['hoorex_data_version'] = HOOREX_VERSION
    PkgDataInfo['deptarget'] = deptarget

    # Step 1 - Create record of all SBo apps and their direct deps.
    #          Save as DirectRequires (which is later extended into PkgRequires)
    for dirpath, dirnames, filenames in os.walk('.'):
        for filenm in filenames:
            if fnmatch.fnmatch(filenm, '*.info'):
                #print(os.path.split(dirpath)[0].strip('.').strip('/'))
                #print(os.path.join(dirpath, filenm))
                category = os.path.split(dirpath)[0].strip('.').strip('/')
                pkgname = os.path.split(dirpath)[-1]
                if pkgname + '.info' == filenm:
                    with open(os.path.join(dirpath, filenm), 'rb') as f:
                        txt = f.read()
                    m = reg.search(txt)
                    if m:
                        value = ''
                        if m.group('value'):
                            # Remove line continuation backslashes
                            value = m.group('value').replace('\\', '')
                        hlog.debug("Adding %s ----- (%s)" % (pkgname, value))
                        PkgCategory[pkgname] = category
                        DirectRequires[pkgname] = value.split()
                        # Expand any $REQUIRES in the list (as may be the case with some other DEPTARGET)
                        if '$REQUIRES' in DirectRequires[pkgname]:
                            requires_reg = reg_requires
                            m = requires_reg.search(txt)
                            if m:
                                value = ''
                                if m.group('value'):
                                    # Remove line continuation backslashes
                                    value = m.group('value').replace('\\', '')
                                hlog.debug("Adding %s ----- (%s)" % (pkgname, value))
                                DirectRequires[pkgname].remove('$REQUIRES')
                                DirectRequires[pkgname].extend(value.split())
                        try:
                            DirectRequires[pkgname].remove('%README%')
                        except:
                            pass

    #print
    hlog.debug("Step 1 done - %d entries" % len(DirectRequires))

    # Step 2 - Find extended requirements by traversing dependency tree.
    #          Save result as PkgRequires
    for (k,v) in DirectRequires.iteritems():
        multilevel_depth = 0
        targets = v
        multi_list = []
        multi_list.extend(targets)
        while True:
            if len(targets) < 1:
                break

            pkglist = []
            for package in targets:
                if DirectRequires.has_key(package):
                    pkglist.extend(DirectRequires[package])

            if len(pkglist) > 0:
                multi_list.extend(pkglist)
            targets = list(set(pkglist))
            try:
                targets.remove('%README%')
            except:
                pass
            multilevel_depth += 1
            if multilevel_depth > DEPTH_LIMIT:
                break

        PkgRequires[k] = multi_list
    hlog.debug("Step 2 done - %d entries" % len(PkgRequires))


    # Step 3 - Generate record of which other pkgs require each SBo pkg
    #          Save as PkgNeededBy
    for k, v in DirectRequires.iteritems():
        #print("%s ----- %s" % (k, v))
        if len(v) > 0:
            #print("%s ----- %s" % (k, v))
            for required in v:
                if not PkgNeededBy.has_key(required):
                    PkgNeededBy[required] = []
                PkgNeededBy[required].extend(k.split())
    hlog.debug("Step 3 done - %d entries" % len(PkgNeededBy))

    PkgData['PkgDataInfo'] = PkgDataInfo
    PkgData['DirectRequires'] = DirectRequires
    PkgData['PkgRequires'] = PkgRequires
    PkgData['PkgNeededBy'] = PkgNeededBy
    PkgData['PkgCategory'] = PkgCategory

    os.chdir(here)


def load_user_config():
    # Try to load the user's configuration
    # Create one if it doesn't already exist

    config_dir = os.path.join(xdg_config_home, appname)
    config_file = os.path.join(config_dir, 'defaults.cfg')
    user_config = ConfigParser.ConfigParser()
    dirty = False

    try:
        os.makedirs(config_dir)
    except:
        # ATM assume any exception is because the path already exists
        pass

    if not os.path.exists(config_file):
        dirty = True
    else:
        user_config.read(config_file)

    if not user_config.has_section('HooRex'):
        dirty = True
        user_config.add_section('HooRex')

    if not user_config.has_option('HooRex', 'force'):
        dirty = True
        user_config.set('HooRex', 'force', 'False')

    if not user_config.has_option('HooRex', 'output_column'):
        dirty = True
        user_config.set('HooRex', 'output_column', 'False')

    if not user_config.has_option('HooRex', 'output_long'):
        dirty = True
        user_config.set('HooRex', 'output_long', 'False')

    if not user_config.has_option('HooRex', 'output_installed'):
        dirty = True
        user_config.set('HooRex', 'output_installed', 'False')

    if not user_config.has_option('HooRex', 'multilevel'):
        dirty = True
        user_config.set('HooRex', 'multilevel', 'False')

    if not user_config.has_option('HooRex', 'reverse_lookup'):
        dirty = True
        user_config.set('HooRex', 'reverse_lookup', 'False')

    if not user_config.has_option('HooRex', 'unknown_strict'):
        dirty = True
        user_config.set('HooRex', 'unknown_strict', 'False')

    if not user_config.has_option('HooRex', 'debug'):
        dirty = True
        user_config.set('HooRex', 'debug', 'False')

    if not user_config.has_option('HooRex', 'slackware_repo'):
        dirty = True
        user_config.set('HooRex', 'slackware_repo', '/var/log/packages')

    if dirty:
        with open(config_file, 'wb') as configfile:
            user_config.write(configfile)

    return user_config


def find_sbo_path():
    hlog.critical("SBO_PATH is not set. Please set it using the -s (--slackbuilds) option e.g.\n hoorex -s /home/thomas/slackbuilds")
    sys.exit(1)

def user_config_set(user_config, key, value):
    config_dir = os.path.join(xdg_config_home, appname)
    config_file = os.path.join(config_dir, 'defaults.cfg')

    # We assume we have been given a valid key & value
    user_config.set('HooRex', key, value)
    with open(config_file, 'wb') as configfile:
        user_config.write(configfile)
    configfile.close()

def sort_output(rawpkgdata, PkgRequires):
    orderedpkgdata = []

    # Continuously cycle through rawpkgdata
    # looking for an element with no dependencies still in rawpkgdata
    # at which time it is removed and added to the sorted list.
    # i.e. packages which are now strictly dependencies of others still
    # in the list (i.e. have no deps of their own in the rawpkgdata list)
    # are moved to sorted list before those which depend on them.
    # The sorted list can be guaranteed to have no member appearing in it
    # before any of its dependencies
    while True:
        for x in range(0, len(rawpkgdata)):
            if x >= len(rawpkgdata):
                continue

            has_dep = False
            prereqs = rawpkgdata[x].split() + PkgRequires[os.path.split(rawpkgdata[x])[-1]]
            for y in range(0, len(rawpkgdata)):
                #hlog.debug("Looking for %s in %s prereqs: %s" %(rawpkgdata[y], rawpkgdata[x], prereqs))
                if not rawpkgdata[y] == rawpkgdata[x] and rawpkgdata[y] in prereqs:
                    has_dep = True
                    hlog.debug("%s still needed (in %s)" %(rawpkgdata[y], rawpkgdata[x]))
                    continue
            if not has_dep:
                hlog.debug("Moving %s to sorted queue" % rawpkgdata[x])
                orderedpkgdata.append(rawpkgdata.pop(x))

        if len(rawpkgdata) == 0:
            break

    return orderedpkgdata


if __name__ == '__main__':
    main()