concat_files.py

#!/usr/bin/env python3
import re
import os
import time
import json
import binascii

import argparse
import hashlib
import subprocess
import datetime


parser = argparse.ArgumentParser(description="Concatenate files to a target file, optionally writing target-changes back to source files")

parser.add_argument('config_file', metavar='CONFIG_FILE', help='Configuration file for concat')
parser.add_argument('-o', '--output', metavar='OUTPUT', help='Override output filename')
parser.add_argument('-w', '--watch', action='store_true', help='Watch files for any changes and map them between the target and source files')
parser.add_argument('-r', '--release', action='store_true', help='Build a version without the section dividers')


class FileWatcher(object):
    def __init__(self, file_list=[]):
        self.file_list = file_list
        self._mtimes = self._get_mtimes()

    def _get_mtimes(self):
        return { filename : os.path.getmtime(filename) for filename in self.file_list if os.path.exists(filename) }

    def get_changes(self):
        mtimes = self._get_mtimes()
        changes = [ filename for filename in self.file_list if self._mtimes.get(filename, 0) < mtimes.get(filename, 0) ]
        self._mtimes.update(mtimes)
        return changes


class FileSection(object):
    def __init__(self, filename, content, modified):
        self.filename = filename
        self.content = content
        self.modified = modified
        self.hash = None

        self.old_hash = None

        if self.content:
            self.recalculate_hash()

    def __repr__(self):
        hash_part = binascii.hexlify(self.hash).decode()[:7]
        if self.old_hash:
            hash_part += ' (' + binascii.hexlify(self.old_hash).decode()[:7] + ')'

        return '<{} \'{}\' {}b {}>'.format(self.__class__.__name__, self.filename, len(self.content), hash_part)

    def recalculate_hash(self):
        self.hash = hashlib.sha256(self.content.encode('utf-8')).digest()
        return self.hash

    @classmethod
    def from_file(cls, filename):
        modified_time = os.path.getmtime(filename)
        with open(filename, 'r') as in_file:
            content = in_file.read()

        return cls(filename=filename,
                   content=content,
                   modified=modified_time)
                   # hash=hash)


class Concatter(object):
    SECTION_REGEX_BASE = r'FileConcat-([SE]) (.+?) HASH:(.+?)'
    SECTION_HEADER_FORMAT_BASE = 'FileConcat-{kind} {filename} HASH:{hash}'
    def __init__(self, config, working_directory=''):
        self.output_filename = config.get('output', 'output.txt')
        self.file_list = config.get('files', [])
        self.section_header_prefix = config.get('header_prefix', '')
        self.section_header_suffix = config.get('header_suffix', '')
        self.working_directory = working_directory

        self.version_metafile = None

        self.newline = None

        self.section_header_format = self.section_header_prefix + self.SECTION_HEADER_FORMAT_BASE + self.section_header_suffix
        self.section_header_regex = re.compile(r'^' + re.escape(self.section_header_prefix) + self.SECTION_REGEX_BASE + re.escape(self.section_header_suffix) + r'$')


    def split_output_file(self):
        file_sections = []
        if not os.path.exists(self.output_filename):
            return file_sections

        modified_time = os.path.getmtime(self.output_filename)
        with open(self.output_filename, 'r') as in_file:
            current_section = None
            section_lines = []

            for line in in_file:
                header_match = self.section_header_regex.match(line)
                if header_match:
                    is_start = header_match.group(1) == 'S'
                    section_filename = header_match.group(2)
                    section_hash = binascii.unhexlify(header_match.group(3))

                    if is_start and current_section is None:
                        current_section = FileSection(section_filename, None, modified_time)
                    elif not is_start and current_section:
                        current_section.content = ''.join(section_lines)
                        current_section.recalculate_hash()
                        current_section.old_hash = section_hash

                        file_sections.append(current_section)

                        current_section = None
                        section_lines = []
                else:
                    section_lines.append(line)

            if current_section is not None:
                raise Exception('Missing file end marker! For ' + current_section.filename)

        return file_sections


    def read_source_files(self):
        file_sections = []

        for filename in self.file_list:
            # The version metafile
            if filename == '<version>':
                file_section = self.version_metafile
            else:
                file_path = os.path.join(self.working_directory, filename)
                if not os.path.exists(file_path):
                    raise Exception("File '{}' is missing!".format(file_path))

                file_section = FileSection.from_file(file_path)
                file_section.filename = filename

                # Figure out newline to be used
                if self.newline is None:
                    self.newline = '\r\n' if '\r\n' in file_section.content else '\n'

                if not file_section.content.endswith(self.newline):
                    file_section.content += self.newline
                    file_section.recalculate_hash()

            file_sections.append(file_section)
        return file_sections


    def concatenate_file_sections(self, file_sections, insert_section_headers=True):
        with open(self.output_filename, 'w') as out_file:
            for file_section in file_sections:
                if insert_section_headers:
                    section_hash = binascii.hexlify(file_section.recalculate_hash()).decode()

                    out_file.write(self.section_header_format.format(kind='S', filename=file_section.filename, hash=section_hash) + self.newline)
                    out_file.write(file_section.content)
                    out_file.write(self.section_header_format.format(kind='E', filename=file_section.filename, hash=section_hash) + self.newline)
                else:
                    out_file.write(file_section.content)


    def write_file_sections_back(self, file_sections):
        for file_section in file_sections:
            # Skip version metafile
            if file_section is self.version_metafile: continue

            file_path = os.path.join(self.working_directory, file_section.filename)

            # Backup target file if it exists
            if os.path.exists(file_path):
                bak_filename = file_path + '.bak'
                if os.path.exists(bak_filename):
                    os.remove(bak_filename)
                os.rename(file_path, bak_filename)

            # Write contents
            with open(file_path, 'w') as out_file:
                out_file.write(file_section.content)


    def _map_sections(self, source_sections, target_sections):
        target_map = {s.filename: s for s in target_sections}

        source_to_target = []
        target_to_source = []

        for source_section in source_sections:
            if source_section is self.version_metafile: continue

            target_section = target_map.get(source_section.filename)

            if not target_section:
                # Target doesn't have this section at all (or is completely empty)
                source_to_target.append(source_section) # Write section to target file
            else:
                source_section.old_hash = target_section.old_hash
                source_newer = source_section.modified > target_section.modified

                source_changed = source_section.hash != source_section.old_hash
                target_changed = target_section.hash != target_section.old_hash

                # Target and source differ
                if target_section.hash != source_section.hash:

                    if source_changed:
                        source_to_target.append(source_section)
                    elif (source_section.hash == target_section.old_hash and source_newer):
                        # Source identical to old target, but file is newer
                        # This happens when you edit the target and then revert the edit on the (changed) source
                        source_section.old_hash = None # Hack (?)
                        source_to_target.append(source_section)
                    else: # Target has been changed
                        target_section.old_hash = target_section.hash # Hack to skip target rewrite
                        source_to_target.append(target_section) # Write section to target file
                        target_to_source.append(target_section) # Write section to source file

                else: # Target and source are identical
                    # source_section.old_hash = target_section.hash # Required?
                    source_to_target.append(source_section) # Write section to target file

        return source_to_target, target_to_source


    def process_changes_in_files(self):
        source_sections = self.read_source_files()
        target_sections = self.split_output_file()

        source_to_target, target_to_source = self._map_sections(source_sections, target_sections)

        changed_sections = [s for s in source_to_target if s.hash != s.old_hash]
        if changed_sections:
            self.concatenate_file_sections(source_to_target)

        if target_to_source:
            self.write_file_sections_back(target_to_source)

        return changed_sections, target_to_source

    def plain_concat(self):
        source_sections = self.read_source_files()
        self.concatenate_file_sections(source_sections, False)


def _create_version_metafile(config, config_dirname):
    repo_dir = os.path.join(config_dirname, config.get('repo_dir', '.'))
    try:
        git_branch = subprocess.check_output(['git', '-C', repo_dir, 'symbolic-ref', '--short', '-q', 'HEAD'], stderr=subprocess.DEVNULL).decode().strip()
        git_commit = subprocess.check_output(['git', '-C', repo_dir, 'rev-parse', '--short', '-q', 'HEAD'], stderr=subprocess.DEVNULL).decode().strip()
    except Exception as e:
        git_branch = None
        git_commit = None

    if not git_branch:
        git_branch = 'unknown'

    if git_commit:
        git_commit_short = git_commit[:7]
    else:
        git_commit = git_commit_short = 'unknown'

    project_version_file = config.get('version_file')
    if project_version_file:
        with open(project_version_file, 'r') as in_file:
            project_version = in_file.read().strip()
    else:
        project_version = 'unknown'

    template_data = {
        'version' : project_version,

        'branch' : git_branch,
        'commit' : git_commit,
        'commit_short' : git_commit_short,

        'now' : datetime.datetime.now(),
        'utc_now' : datetime.datetime.utcnow(),
    }

    version_template_file = config.get('version_template_file')
    if version_template_file:
        with open(os.path.join(config_dirname, version_template_file), 'r') as in_file:
            version_template = in_file.read()
    else:
        version_template = ''

    version_metafile = FileSection('<version>', version_template.format(**template_data), 0)
    return version_metafile


def _print_change_writes(source_to_target, target_to_source):
    if source_to_target:
        print('SOURCE -> TARGET')
        print(source_to_target)
    if target_to_source:
        print('TARGET -> SOURCE')
        print(target_to_source)
    if not source_to_target and not target_to_source:
        print('No changes.')


if __name__ == '__main__':
    args = parser.parse_args()

    if not os.path.exists(args.config_file):
        print('Unable to find given configuration file \'{}\''.format(args.config_file))
        exit(1)

    try:
        with open(args.config_file, 'r') as in_file:
            config = json.load(in_file)
    except:
        print('Unable to read given configuration file \'{}\''.format(args.config_file))
        exit(1)

    config_dirname = os.path.dirname(args.config_file)
    if args.output:
        config['output'] = args.output
    else:
        # Make output be relative to config file
        config['output'] = os.path.join(config_dirname, config['output'])

    concatter = Concatter(config, config_dirname)
    concatter.version_metafile = _create_version_metafile(config, config_dirname)

    if not concatter.file_list:
        print('No files listed in configuration!')
        exit(1)

    if not args.watch:
        if args.release:
            concatter.plain_concat()
            print("Concatenated source files to '{}'".format(concatter.output_filename))
        else:
            s2t, t2s = concatter.process_changes_in_files()
            _print_change_writes(s2t, t2s)
    else:
        tracked_files_list = list(concatter.file_list)
        tracked_files_list.append(concatter.output_filename)

        file_watcher = FileWatcher(tracked_files_list)

        print('Watching changes for', len(tracked_files_list), 'files...')
        while True:
            changes = file_watcher.get_changes()

            if changes:
                print("------------------------", changes)
                if args.release:
                    concatter.plain_concat()
                    print("Concatenated source files to '{}'".format(concatter.output_filename))
                else:
                    s2t, t2s = concatter.process_changes_in_files()
                    _print_change_writes(s2t, t2s)
                # Grab new mtimes
                file_watcher.get_changes()

            time.sleep(0.25)