linkulator2/data.py

#!/usr/bin/env python3                
"""This module takes input and returns link_data, the data structure linkulator works from"""
from time import time
from pathlib import PurePath
from glob import glob
import re

# regex for removing escape characters from https://stackoverflow.com/a/14693789
ESCAPE_CHARS = re.compile(r"\x1B[@-_][0-?]*[ -/]*[@-~]")
BAD_CHARS = re.compile(r"[\t\r\n\f\v]*")


def is_well_formed_line(line: str) -> bool:
    """Checks if current line is valid or not, returns true and false respectively."""
    pipe_count = (
        4  ## A PROPERLY FORMATED LINE IN linkulator.data HAS EXACTLY FOUR PIPES.
    )
    return line.count("|") == pipe_count


def is_valid_time(timestamp: str) -> bool:
    """identifies future dated timestamps - returns true if valid time, false is invalid"""
    return float(timestamp) < time()


def wash_line(line: str) -> str:
    """take line and return a version with bad characters removed"""
    line = ESCAPE_CHARS.sub("", line)
    line = BAD_CHARS.sub("", line)
    return line


def process(line: str, file_owner: str):
    """Takes a line, returns a list based on the delimeter pipe character"""
    if not is_well_formed_line(line):
        raise ValueError("Not a well formed record")
    line = wash_line(line)
    split_line = line.split("|")
    if split_line[0] and not is_valid_time(split_line[0]):
        raise ValueError("Invalid date")
    split_line.insert(0, file_owner)
    return split_line


def get(config, ignore_names):
    """reads data files for non-ignored users and returns valid data in linkulator formats"""
    link_data = []
    ## username, datestamp, parent-id, category, link-url, link-title
    categories = []
    category_counts = {}
    ignore_names = []

    ## WHENEVER THIS FUNCTION IS CALLED, THE DATA IS REFRESHED FROM FILES. SINCE
    ## DISK IO IS PROBABLY THE HEAVIEST PART OF THIS SCRIPT, DON'T DO THIS OFTEN.

    files_pattern = str(
        PurePath(config.PATHS.all_homedir_pattern).joinpath(
            config.PATHS.datadir, config.PATHS.datafile
        )
    )
    linkulator_files = glob(files_pattern)

    id_iterator = 1

    for filename in linkulator_files:
        with open(filename) as cfile:
            # get file owner username from path
            file_owner = PurePath(filename).parent.parent.name
            if file_owner in ignore_names:
                # ignore names found in ignore file
                continue
            for line in cfile:
                try:
                    split_line = process(line, file_owner)
                except ValueError:
                    continue

                # assign parent items (links) an ID
                if split_line[2] == "":
                    split_line.insert(0, id_iterator)
                    id_iterator += 1
                else:
                    split_line.insert(0, "")

                link_data.append(split_line)

    # sort links by creation date
    link_data.sort(key=lambda x: x[2], reverse=True)

    # generate categories list and category count from sorted link data
    for record in link_data:
        cat = record[4]
        if cat not in categories and cat != "":
            categories.append(cat)
            category_counts[cat] = 1
        elif cat in categories:
            category_counts[cat] += 1

    return link_data, categories, category_counts