import abc
import base64
import binascii
import hashlib
import json
import os
import os.path
import re
import shlex
import stat
import unicodedata
import uuid
from pathlib import Path
from typing import ClassVar, Any, TYPE_CHECKING, Literal
from datetime import UTC, datetime, timedelta
from functools import partial
from hashlib import sha256
from string import Formatter

from ..logger import create_logger

logger = create_logger()

import yaml
from blake3 import blake3

from .errors import Error, CommandError
from .fs import make_path_safe, slashify
from .argparsing import Action, ArgumentError, ArgumentTypeError, register_type
from .msgpack import Timestamp
from .time import OutputTimestamp, format_time, safe_timestamp
from .. import __version__ as borg_version
from .. import __version_tuple__ as borg_version_tuple
from ..constants import *  # NOQA
from ..platformflags import is_win32, is_darwin

if TYPE_CHECKING:
    from ..item import ItemDiff


def octal_int(s):
    if isinstance(s, int):
        return s
    return int(s, 8)


def bin_to_hex(binary):
    return binascii.hexlify(binary).decode("ascii")


def hex_to_bin(hex, length=None):
    try:
        binary = binascii.unhexlify(hex)
        binary_len = len(binary)
        if length is not None and binary_len != length:
            raise ValueError(f"Expected {length} bytes ({2 * length} hex digits), got {binary_len} bytes.")
    except binascii.Error as e:
        raise ValueError(str(e)) from None
    return binary


def safe_decode(s, coding="utf-8", errors="surrogateescape"):
    """Decode bytes to str, with round-tripping of "invalid" bytes."""
    if s is None:
        return None
    return s.decode(coding, errors)


def safe_encode(s, coding="utf-8", errors="surrogateescape"):
    """Encode str to bytes, with round-tripping of "invalid" bytes."""
    if s is None:
        return None
    return s.encode(coding, errors)


def remove_surrogates(s, errors="replace"):
    """Replace surrogates generated by fsdecode with '?'."""
    return s.encode("utf-8", errors).decode("utf-8")


def binary_to_json(key, value):
    assert isinstance(key, str)
    assert isinstance(value, bytes)
    return {key + "_b64": base64.b64encode(value).decode("ascii")}


def text_to_json(key, value):
    """
    Return a dict made from key/value that can be fed safely into a JSON encoder.

    JSON can only contain pure, valid unicode (but not: unicode with surrogate escapes).

    But sometimes we have to deal with such values and we do it like this:
    - <key>: value as pure unicode text (surrogate escapes, if any, replaced by ?)
    - <key>_b64: value as base64 encoded binary representation (only set if value has surrogate-escapes)
    """
    coding = "utf-8"
    assert isinstance(key, str)
    assert isinstance(value, str)  # str might contain surrogate escapes
    data = {}
    try:
        value.encode(coding, errors="strict")  # check if pure unicode
    except UnicodeEncodeError:
        # value has surrogate escape sequences
        data[key] = remove_surrogates(value)
        value_bytes = value.encode(coding, errors="surrogateescape")
        data |= binary_to_json(key, value_bytes)
    else:
        # value is pure unicode
        data[key] = value
        # we do not give the b64 representation, not needed
    return data


def join_cmd(argv, rs=False):
    cmd = shlex.join(argv)
    return remove_surrogates(cmd) if rs else cmd


def eval_escapes(s):
    """Evaluate literal escape sequences in a string (e.g. `\\n` -> `\n`)."""
    return s.encode("ascii", "backslashreplace").decode("unicode-escape")


def decode_dict(d, keys, encoding="utf-8", errors="surrogateescape"):
    for key in keys:
        if isinstance(d.get(key), bytes):
            d[key] = d[key].decode(encoding, errors)
    return d


def nonnegative_seconds(s):
    """Parse a nonnegative number of seconds (int or float, e.g. ``0``, ``30`` or ``2.5``)."""
    try:
        seconds = float(s)
    except ValueError:
        raise ArgumentTypeError(f"invalid seconds value: {s!r}") from None
    if not seconds >= 0.0:
        raise ArgumentTypeError("seconds value must be >= 0")
    return seconds


def interval(s):
    """Parse an interval string (e.g. ``7d``, ``2w``, ``30M``) into a timedelta.

    Supported units: y (years, 365d), m (months, 31d), w (weeks), d (days),
    H (hours), M (minutes), S (seconds).  The value must be nonnegative.
    """
    if isinstance(s, timedelta):
        return s

    seconds_in_a_minute = 60
    seconds_in_an_hour = 60 * seconds_in_a_minute
    seconds_in_a_day = 24 * seconds_in_an_hour
    seconds_in_a_week = 7 * seconds_in_a_day
    seconds_in_a_month = 31 * seconds_in_a_day
    seconds_in_a_year = 365 * seconds_in_a_day
    multiplier = dict(
        y=seconds_in_a_year,
        m=seconds_in_a_month,
        w=seconds_in_a_week,
        d=seconds_in_a_day,
        H=seconds_in_an_hour,
        M=seconds_in_a_minute,
        S=1,
    )

    if s.endswith(tuple(multiplier.keys())):
        number = s[:-1]
        suffix = s[-1]
    else:
        raise ArgumentTypeError(f'Unexpected time unit "{s[-1]}": choose from {", ".join(multiplier)}')

    try:
        seconds = int(number) * multiplier[suffix]
    except ValueError:
        seconds = -1

    if seconds < 0:
        raise ArgumentTypeError(f'Invalid number "{number}": expected nonnegative integer')

    return timedelta(seconds=seconds)


def int_or_interval(s):
    """Parse *s* as an :class:`int` or, failing that, as an interval string.

    Returns :class:`int` if *s* can be parsed as an integer (e.g. ``"7"``),
    or :class:`datetime.timedelta` if *s* is a valid interval (e.g. ``"7d"``).

    Raises :class:`ArgumentTypeError` if *s* is neither an integer nor
    a valid interval.
    """
    if isinstance(s, (int, timedelta)):
        return s

    # Explicitly check 'all' as a shortcut to 'infinite' sentinel value `-1`.
    if s == "all":
        return -1

    try:
        return int(s)
    except ValueError:
        pass

    try:
        return interval(s)
    except ArgumentTypeError as e:
        raise ArgumentTypeError(f"Value is neither an integer nor an interval: {e}")


class CompressionSpec:
    def __init__(self, s):
        if isinstance(s, CompressionSpec):
            self.__dict__.update(s.__dict__)
            return
        values = s.split(",")
        count = len(values)
        if count < 1:
            raise ArgumentTypeError("not enough arguments")
        # --compression algo[,level]
        self.name = values[0]
        match self.name:
            case "none" | "lz4":
                return
            case "zlib" | "lzma" | "zlib_legacy":  # zlib_legacy just for testing
                if count < 2:
                    level = 6  # default compression level in py stdlib
                elif count == 2:
                    level = int(values[1])
                    if not 0 <= level <= 9:
                        raise ArgumentTypeError("level must be >= 0 and <= 9")
                else:
                    raise ArgumentTypeError("too many arguments")
                self.level = level
            case "zstd":
                if count < 2:
                    level = 3  # default compression level in zstd
                elif count == 2:
                    level = int(values[1])
                    # negative levels are zstd's "fast" levels, -128 is what the clevel byte
                    # can still store (it is interpreted as an int8_t for zstd).
                    if not -128 <= level <= 22:
                        raise ArgumentTypeError("level must be >= -128 and <= 22")
                else:
                    raise ArgumentTypeError("too many arguments")
                self.level = level
            case "auto":
                if 2 <= count <= 3:
                    compression = ",".join(values[1:])
                else:
                    raise ArgumentTypeError("bad arguments")
                self.inner = CompressionSpec(compression)
            case "obfuscate":
                if 3 <= count <= 5:
                    level = int(values[1])
                    if not ((1 <= level <= 6) or (110 <= level <= 123) or (level == 250)):
                        raise ArgumentTypeError("level must be (inclusively) within 1...6, 110...123 or equal to 250")
                    self.level = level
                    compression = ",".join(values[2:])
                else:
                    raise ArgumentTypeError("bad arguments")
                self.inner = CompressionSpec(compression)
            case _:
                raise ArgumentTypeError("unsupported compression type")

    @property
    def compressor(self):
        from ..compress import get_compressor

        match self.name:
            case "none" | "lz4":
                return get_compressor(self.name)
            case "zlib" | "lzma" | "zstd" | "zlib_legacy":
                return get_compressor(self.name, level=self.level)
            case "auto":
                return get_compressor(self.name, compressor=self.inner.compressor)
            case "obfuscate":
                return get_compressor(self.name, level=self.level, compressor=self.inner.compressor)

    def __str__(self):
        match self.name:
            case "none" | "lz4":
                return f"{self.name}"
            case "zlib" | "lzma" | "zstd" | "zlib_legacy":
                return f"{self.name},{self.level}"
            case "auto":
                return f"auto,{self.inner}"
            case "obfuscate":
                return f"obfuscate,{self.level},{self.inner}"
            case _:
                raise ValueError(f"unsupported compression type: {self.name}")


def ChunkerParams(s):
    if isinstance(s, (list, tuple)):
        return tuple(s)
    params = s.strip().split(",")
    count = len(params)
    if count == 0:
        raise ArgumentTypeError("no chunker params given")
    algo = params[0].lower()
    if algo == CH_FAIL and count == 3:
        block_size = int(params[1])
        fail_map = str(params[2])
        return algo, block_size, fail_map
    if algo == CH_FIXED and 2 <= count <= 3:  # fixed, block_size[, header_size]
        block_size = int(params[1])
        header_size = int(params[2]) if count == 3 else 0
        if block_size < 64:
            # we are only disallowing the most extreme cases of abuse here - this does NOT imply
            # that cutting chunks of the minimum allowed size is efficient concerning storage
            # or in-memory chunk management.
            # choose the block (chunk) size wisely: if you have a lot of data and you cut
            # it into very small chunks, you are asking for trouble!
            raise ArgumentTypeError("block_size must not be less than 64 Bytes")
        if block_size > MAX_DATA_SIZE or header_size > MAX_DATA_SIZE:
            raise ArgumentTypeError("block_size and header_size must not exceed MAX_DATA_SIZE [%d]" % MAX_DATA_SIZE)
        return algo, block_size, header_size
    if algo == "default" and count == 1:  # default
        return CHUNKER_PARAMS
    if algo == CH_BUZHASH64:
        # buzhash64, chunk_min, chunk_max, chunk_mask, window_size, nc_level
        # use nc_level 0 to disable normalized chunking.
        if count != 6:
            raise ArgumentTypeError(
                "buzhash64 chunker params must be: buzhash64,chunk_min,chunk_max,chunk_mask,window_size,nc_level"
            )
        chunk_min, chunk_max, chunk_mask, window_size = (int(p) for p in params[1:5])
        nc_level = int(params[5])
        if not (chunk_min <= chunk_mask <= chunk_max):
            raise ArgumentTypeError("required: chunk_min <= chunk_mask <= chunk_max")
        if chunk_min < 6:
            # see comment in 'fixed' algo check
            raise ArgumentTypeError("min. chunk size exponent must not be less than 6 (2^6 = 64B min. chunk size)")
        if chunk_max > 23:
            raise ArgumentTypeError("max. chunk size exponent must not be more than 23 (2^23 = 8MiB max. chunk size)")
        # the chunker needs the hash window plus one byte beyond the min. chunk size to fit into
        # a max. chunk size sized buffer, see the assertion in ChunkerBuzHash64.__cinit__.
        if window_size + 2**chunk_min + 1 > 2**chunk_max:
            raise ArgumentTypeError("required: window_size + 2^chunk_min + 1 <= 2^chunk_max")
        # normalized chunking switches the mask at the target size; it needs room below and above
        # the base mask bits (chunk_mask). nc_level 0 disables it.
        if not (0 <= nc_level and chunk_mask - nc_level >= 1 and chunk_mask + nc_level <= 48):
            raise ArgumentTypeError(
                "required: 0 <= nc_level and 1 <= chunk_mask - nc_level and chunk_mask + nc_level <= 48"
            )
        # note that for buzhash64, there is no problem with even window_size.
        return CH_BUZHASH64, chunk_min, chunk_max, chunk_mask, window_size, nc_level
    if algo == CH_FASTCDC:
        # fastcdc, chunk_min, chunk_max, chunk_mask, nc_level
        # fastcdc uses a window-less Gear hash, so there is no window_size field.
        # nc_level is required; use nc_level 0 to disable normalized chunking.
        if count != 5:
            raise ArgumentTypeError("fastcdc chunker params must be: fastcdc,chunk_min,chunk_max,chunk_mask,nc_level")
        chunk_min, chunk_max, chunk_mask = (int(p) for p in params[1:4])
        nc_level = int(params[4])
        if not (chunk_min <= chunk_mask <= chunk_max):
            raise ArgumentTypeError("required: chunk_min <= chunk_mask <= chunk_max")
        if chunk_min < 6:
            # see comment in 'fixed' algo check
            raise ArgumentTypeError("min. chunk size exponent must not be less than 6 (2^6 = 64B min. chunk size)")
        if chunk_max > 23:
            raise ArgumentTypeError("max. chunk size exponent must not be more than 23 (2^23 = 8MiB max. chunk size)")
        # the chunker needs one byte beyond the min. chunk size to fit into a max. chunk size
        # sized buffer, see the assertion in ChunkerFastCDC.__cinit__.
        if chunk_min >= chunk_max:
            raise ArgumentTypeError("required: chunk_min < chunk_max")
        if not (0 <= nc_level and chunk_mask - nc_level >= 1 and chunk_mask + nc_level <= 48):
            raise ArgumentTypeError(
                "required: 0 <= nc_level and 1 <= chunk_mask - nc_level and chunk_mask + nc_level <= 48"
            )
        return CH_FASTCDC, chunk_min, chunk_max, chunk_mask, nc_level
    if algo in (CH_RABIN_AES, CH_GOLDILOCKS_AES, CH_TOEPLITZ_AES):
        # <algo>, chunk_min, chunk_max, chunk_mask, nc_level
        # both UHF-then-PRF chunkers have a fixed 64-byte window, so there is no window_size field.
        # nc_level is required; use nc_level 0 to disable normalized chunking.
        if count != 5:
            raise ArgumentTypeError(f"{algo} chunker params must be: {algo},chunk_min,chunk_max,chunk_mask,nc_level")
        chunk_min, chunk_max, chunk_mask = (int(p) for p in params[1:4])
        nc_level = int(params[4])
        if not (chunk_min <= chunk_mask <= chunk_max):
            raise ArgumentTypeError("required: chunk_min <= chunk_mask <= chunk_max")
        if chunk_min < 6:
            # the rolling window needs 64 bytes of in-chunk history at the first possible
            # cut position, so the min. chunk size must be at least the 64-byte window size
            # (this also matches the general lower bound, see comment in 'fixed' algo check).
            raise ArgumentTypeError("min. chunk size exponent must not be less than 6 (2^6 = 64B min. chunk size)")
        if chunk_max > 23:
            raise ArgumentTypeError("max. chunk size exponent must not be more than 23 (2^23 = 8MiB max. chunk size)")
        if chunk_min >= chunk_max:
            raise ArgumentTypeError("required: chunk_min < chunk_max")
        if not (0 <= nc_level and chunk_mask - nc_level >= 1 and chunk_mask + nc_level <= 48):
            raise ArgumentTypeError(
                "required: 0 <= nc_level and 1 <= chunk_mask - nc_level and chunk_mask + nc_level <= 48"
            )
        return algo, chunk_min, chunk_max, chunk_mask, nc_level
    # this must stay last as it deals with old-style compat mode (no algorithm, 4 numeric params, buzhash);
    # the isdigit check keeps misspelled/incomplete algo specs out of the compat branch.
    if (algo == CH_BUZHASH and count == 5) or (
        count == 4 and all(p.strip().isdigit() for p in params)
    ):  # [buzhash, ]chunk_min, chunk_max, chunk_mask, window_size
        chunk_min, chunk_max, chunk_mask, window_size = (int(p) for p in params[count - 4 :])
        if not (chunk_min <= chunk_mask <= chunk_max):
            raise ArgumentTypeError("required: chunk_min <= chunk_mask <= chunk_max")
        if chunk_min < 6:
            # see comment in 'fixed' algo check
            raise ArgumentTypeError("min. chunk size exponent must not be less than 6 (2^6 = 64B min. chunk size)")
        if chunk_max > 23:
            raise ArgumentTypeError("max. chunk size exponent must not be more than 23 (2^23 = 8MiB max. chunk size)")
        if window_size % 2 == 0:
            raise ArgumentTypeError("window_size must be an uneven (odd) number")
        return CH_BUZHASH, chunk_min, chunk_max, chunk_mask, window_size
    raise ArgumentTypeError("invalid chunker params")


def DigestAlgos(s):
    """--digests: parse a comma-separated list of hash algorithm names (or "none")"""
    if isinstance(s, (list, tuple)):  # the default value is given as a tuple already, see ChunkerParams
        return tuple(s)
    if s == "none":
        return ()
    algos = tuple(sorted(set(s.strip().split(","))))
    if not set(algos) <= HASH_ALGORITHMS:
        raise ArgumentTypeError(
            'digests must be "none" or a comma-separated list of: %s' % ",".join(sorted(HASH_ALGORITHMS))
        )
    return algos


def FilesCacheMode(s):
    ENTRIES_MAP = dict(ctime="c", mtime="m", size="s", inode="i", rechunk="r", disabled="d")
    VALID_MODES = ("cis", "ims", "cs", "ms", "cr", "mr", "d", "s")  # letters in alpha order
    if s in VALID_MODES:
        return s
    entries = set(s.strip().split(","))
    if not entries <= set(ENTRIES_MAP):
        raise ArgumentTypeError("cache mode must be a comma-separated list of: %s" % ",".join(sorted(ENTRIES_MAP)))
    short_entries = {ENTRIES_MAP[entry] for entry in entries}
    mode = "".join(sorted(short_entries))
    if mode not in VALID_MODES:
        raise ArgumentTypeError("cache mode short must be one of: %s" % ",".join(VALID_MODES))
    return mode


# ctime based files cache modes and their mtime based equivalent, see #7193.
FILES_CACHE_MODE_CTIME_TO_MTIME = {"cis": "ims", "cs": "ms", "cr": "mr"}


def files_cache_mode_no_ctime(mode):
    """
    Return (new_mode, changed): <mode> with ctime replaced by mtime.

    <mode> may be given in long form ("ctime,size,inode") or in short form ("cis"),
    the returned mode is always the short form, as FilesCacheMode() gives it.
    "changed" tells whether a ctime -> mtime replacement actually happened.
    """
    mode = FilesCacheMode(mode)  # long form -> short form (idempotent for the short form)
    new_mode = FILES_CACHE_MODE_CTIME_TO_MTIME.get(mode, mode)
    return new_mode, new_mode != mode


def partial_format(format, mapping):
    """
    Apply format.format_map(mapping) while preserving unknown keys

    Does not support attribute access, indexing and ![rsa] conversions
    """
    for key, value in mapping.items():
        key = re.escape(key)
        format = re.sub(
            rf"(?<!\{{)((\{{{key}\}})|(\{{{key}:[^\}}]*\}}))", lambda match: match.group(1).format_map(mapping), format
        )
    return format


class DatetimeWrapper:
    def __init__(self, dt):
        self.dt = dt

    def __format__(self, format_spec):
        if format_spec == "":
            format_spec = ISO_FORMAT_NO_USECS
        return self.dt.__format__(format_spec)


class PlaceholderError(Error):
    """Formatting Error: "{}".format({}): {}({})"""

    exit_mcode = 5


class InvalidPlaceholder(PlaceholderError):
    """Invalid placeholder "{}" in string: {}"""

    exit_mcode = 6


def format_line(format, data):
    for _, key, _, conversion in Formatter().parse(format):
        if not key:
            continue
        if conversion or key not in data:
            raise InvalidPlaceholder(key, format)
    try:
        return format.format_map(data)
    except Exception as e:
        raise PlaceholderError(format, data, e.__class__.__name__, str(e))


def _replace_placeholders(text, overrides={}):
    """Replace placeholders in text with their values."""
    from ..platform import get_fqdn, get_hostname, getosusername

    fqdn = get_fqdn()
    current_time = datetime.now(UTC)
    data = {
        "pid": os.getpid(),
        "fqdn": fqdn,
        "reverse-fqdn": ".".join(reversed(fqdn.split("."))),
        "hostname": os.environ.get("BORG_HOSTNAME") or get_hostname(),
        "now": DatetimeWrapper(current_time.astimezone()),
        "utcnow": DatetimeWrapper(current_time),
        "unixtime": int(current_time.timestamp()),
        "user": os.environ.get("BORG_USERNAME") or getosusername(),
        "uuid4": str(uuid.uuid4()),
        "borgversion": borg_version,
        "borgmajor": "%d" % borg_version_tuple[:1],
        "borgminor": "%d.%d" % borg_version_tuple[:2],
        "borgpatch": "%d.%d.%d" % borg_version_tuple[:3],
        **overrides,
    }
    return format_line(text, data)


class PlaceholderReplacer:
    def __init__(self):
        self.reset()

    def override(self, key, value):
        self.overrides[key] = value

    def reset(self):
        self.overrides = {}

    def __call__(self, text, overrides=None):
        ovr = self.overrides | (overrides or {})
        return _replace_placeholders(text, overrides=ovr)


replace_placeholders = PlaceholderReplacer()


def PathSpec(text):
    """A path inside an archive."""
    if not text:
        raise ArgumentTypeError("Empty strings are not accepted as paths.")
    return text


def FilesystemPathSpec(text):
    """A path (file or directory) in the local filesystem."""
    if not text:
        raise ArgumentTypeError("Empty strings are not accepted as paths.")
    return slashify(text)


def FilesystemDirSpec(text):
    """A directory in the local filesystem."""
    if not text:
        raise ArgumentTypeError("Empty strings are not accepted as paths.")
    return slashify(text)


def SortBySpec(text):
    from ..manifest import AI_HUMAN_SORT_KEYS

    for sort_key in text.split(","):
        if sort_key not in AI_HUMAN_SORT_KEYS and sort_key != "ts":  # idempotency: do not reject ts
            raise ArgumentTypeError("Invalid sort key: %s" % sort_key)
    return text.replace("timestamp", "ts").replace("archive", "name")


SIZE_UNITS = ("si", "iec", "raw")

_warned_units: set[str] = set()  # invalid BORG_UNITS values already complained about


def get_size_units():
    """Which units shall sizes be formatted with: "si" (default), "iec" or "raw"? See BORG_UNITS."""
    units = os.environ.get("BORG_UNITS", "").strip().lower()
    if units in SIZE_UNITS:
        return units
    if units and units not in _warned_units:
        _warned_units.add(units)
        logger.warning(f"Invalid BORG_UNITS value {units!r}, must be one of {', '.join(SIZE_UNITS)}. Ignoring it.")
    return "si"


def format_file_size(v, precision=2, sign=False, fine=False):
    """Format file size into a human friendly format, using the units requested via BORG_UNITS.

    If fine is given, big sizes get more decimals, so that changes of ~1MB stay visible.
    """
    units = get_size_units()
    if units == "raw":
        # exact byte counts, so scripts can easily parse the output (fine does not apply)
        v = round(v)  # v might be a float, e.g. a throughput value
        return f"{v:{'+' if sign and v > 0 else ''}d} B"
    fn = sizeof_fmt_iec if units == "iec" else sizeof_fmt_decimal
    return fn(v, suffix="B", sep=" ", precision=precision, sign=sign, fine=fine)


class FileSize(int):
    def __format__(self, format_spec):
        return format_file_size(int(self)).__format__(format_spec)


def parse_file_size(s):
    """Return int from file size (1234, 55G, 1.7T)."""
    if isinstance(s, int):
        return s
    if not s:
        return int(s)  # will raise
    s = s.upper()
    suffix = s[-1]
    power = 1000
    try:
        factor = {"K": power, "M": power**2, "G": power**3, "T": power**4, "P": power**5}[suffix]
        s = s[:-1]
    except KeyError:
        factor = 1
    return int(float(s) * factor)


def sizeof_fmt(num, suffix="B", units=None, power=None, sep="", precision=2, sign=False, fine=False):
    sign = "+" if sign and num > 0 else ""
    fmt = "{0:{1}.{2}f}{3}{4}{5}"
    prec = 0
    for i, unit in enumerate(units[:-1]):
        if abs(round(num, precision)) < power:
            break
        num /= float(power)
        prec = precision
    else:
        i = len(units) - 1
        unit = units[-1]
    if fine:
        # scale dependent precision, so ~1 M<unit> stays visible at any scale: M: 2 (default),
        # G: 3, T: 6, P: 9 decimals. capped at 9 - a float can not resolve more anyway and it
        # bounds the output width. see #3559.
        prec = max(prec, min((i - 2) * 3, 9))
    return fmt.format(num, sign, prec, sep, unit, suffix)


def sizeof_fmt_iec(num, suffix="B", sep="", precision=2, sign=False, fine=False):
    return sizeof_fmt(
        num,
        suffix=suffix,
        sep=sep,
        precision=precision,
        sign=sign,
        fine=fine,
        units=["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi"],
        power=1024,
    )


def sizeof_fmt_decimal(num, suffix="B", sep="", precision=2, sign=False, fine=False):
    return sizeof_fmt(
        num,
        suffix=suffix,
        sep=sep,
        precision=precision,
        sign=sign,
        fine=fine,
        units=["", "k", "M", "G", "T", "P", "E", "Z", "Y"],
        power=1000,
    )


def format_archive(archive):
    return "%-36s %s [%s]" % (archive.name, format_time(archive.ts), bin_to_hex(archive.id))


def parse_stringified_list(s):
    items = re.split(" *, *", s)
    return [item for item in items if item != ""]


def _redact_url_credentials(url):
    """Remove embedded credentials from a repository URL for safe display/identity use."""
    # netloc style: scheme://user[:pass]@host... -> scheme://host...
    url = re.sub(r"(://)[^/@]+@", r"\1", url)
    # s3/b2 style: (s3|b2):profile@... or (s3|b2):key:secret@... -> (s3|b2):...
    url = re.sub(r"^((?:s3|b2):)[^@/]+@", r"\1", url)
    return url


def normalize_local_path(path):
    """Normalize a local filesystem path's unicode form so paths compare equal regardless of how they were entered.

    On macOS the filesystem returns paths in NFD (decomposed) form, while a path typed as a command line
    argument is usually in NFC (composed) form. These look identical but are byte-different, which otherwise
    makes borg believe a repository was relocated (see issue #2913). We normalize to NFD to match what the
    macOS filesystem uses. On other platforms the path is returned unchanged.
    """
    return unicodedata.normalize("NFD", path) if is_darwin else path


class Location:
    """Object representing a repository location"""

    # user@ (optional)
    # user must not contain "@", ":" or "/".
    # Quoting adduser error message:
    # "To avoid problems, the username should consist only of letters, digits,
    # underscores, periods, at signs and dashes, and not start with a dash
    # (as defined by IEEE Std 1003.1-2001)."
    # We use "@" as separator between username and hostname, so we must
    # disallow it within the pure username part.
    optional_user_re = r"(?:(?P<user>[^@:/]+)@)?"

    # host NAME, or host IP ADDRESS (v4 or v6, v6 must be in square brackets)
    host_re = r"""
        (?P<host>(
            (?!\[)[^:/]+(?<!\])     # hostname or v4 addr, not containing : or / (does not match v6 addr: no brackets!)
            |
            \[[0-9a-fA-F:.]+\])     # ipv6 address in brackets
        )
    """

    # :port (optional)
    optional_port_re = r"(?::(?P<port>\d+))?"

    # locations that borgstore parses and validates itself - borg only detects the scheme and
    # passes the raw URL through. covers both "scheme://..." and opaque "scheme:..." forms.
    BORGSTORE_SCHEMES = ("sftp", "http", "https", "s3", "b2", "rclone")

    # path may contain any chars. to avoid ambiguities with other regexes, it must not start with
    # a "scheme://" or one of the borgstore "scheme:" specifiers (all of which are matched
    # before local_re in _parse). the borgstore scheme list is sourced from BORGSTORE_SCHEMES.
    local_path_re = r"(?!((?:ssh|file)://|(?:" + "|".join(BORGSTORE_SCHEMES) + r"):))" r"(?P<path>.+)"

    # abs_path must start with a slash (or drive letter on Windows).
    abs_path_re = r"(?P<path>[A-Za-z]:/.+)" if is_win32 else r"(?P<path>/.+)"

    # path may or may not start with a slash.
    abs_or_rel_path_re = r"(?P<path>.+)"

    # We only parse out individual fields (user/host/port/path) for the protocols where borg
    # itself needs them: legacy "ssh" (v1 repositories) and "rest" (for the ssh tunnel + FILE
    # backend), plus local "file" paths. Everything else (see BORGSTORE_SCHEMES) is handed to
    # borgstore as the raw URL and parsed/validated there - we only detect the scheme.

    # ssh:// is only used for legacy borg 1.x repositories nowadays.
    ssh_re = re.compile(
        r"(?P<proto>ssh)://"
        + optional_user_re
        + host_re
        + optional_port_re
        + r"/"  # this is the separator, not part of the path!
        + abs_or_rel_path_re,
        re.VERBOSE,
    )

    # REST http via stdio (via ssh, if host given):
    rest_re = re.compile(
        r"(?P<proto>(rest))://"
        + r"("
        + optional_user_re
        + host_re
        + optional_port_re
        + r")?"
        + r"/"  # this is the separator, not part of the path!
        + abs_or_rel_path_re,
        re.VERBOSE,
    )

    # scheme detector for the borgstore-handled locations listed in BORGSTORE_SCHEMES above.
    scheme_re = re.compile(r"(?P<scheme>[a-zA-Z][a-zA-Z0-9+.\-]*):")

    sl = "/" if is_win32 else ""
    file_re = re.compile(r"(?P<proto>file)://" + sl + abs_path_re, re.VERBOSE)

    local_re = re.compile(local_path_re, re.VERBOSE)

    def __init__(self, text="", overrides={}, other=False):
        if isinstance(text, Location):
            self.__dict__.update(text.__dict__)
            return
        self.repo_env_var = "BORG_OTHER_REPO" if other else "BORG_REPO"
        self.valid = False
        self.proto = None
        self.user = None
        self._pass = None
        self._host = None
        self.port = None
        self.path = None
        self.raw = None
        self.processed = None
        self.parse(text, overrides)

    def parse(self, text, overrides={}):
        if not text:
            # we did not get a text to parse, so we try to fetch from the environment
            text = os.environ.get(self.repo_env_var)
            if not text:  # None or ""
                return

        self.raw = text  # as given by user, might contain placeholders
        self.processed = replace_placeholders(self.raw, overrides)  # after placeholder replacement
        if not self._parse(self.processed):
            raise ValueError('Invalid location format: "%s"' % self.processed)
        if self.proto == "file" and self.path.startswith("//"):
            # UNC path (//server/share/..., \\server\share\...): not supported, fail early with a hint, see #10164.
            raise ValueError(
                f'UNC paths are not supported: "{self.processed}". '
                "Mount the share (Windows: net use X: \\\\server\\share) and use the mounted path instead."
            )
        self.valid = True

    def _parse(self, text):
        m = self.ssh_re.match(text)
        if m:
            self.proto = m.group("proto")
            self.user = m.group("user")
            self._host = m.group("host")
            self.port = m.group("port") and int(m.group("port")) or None
            self.path = os.path.normpath(m.group("path"))
            return True
        m = self.rest_re.match(text)
        if m:
            self.proto = m.group("proto")
            self.user = m.group("user")
            self._host = m.group("host")
            self.port = m.group("port") and int(m.group("port")) or None
            self.path = os.path.normpath(m.group("path"))
            return True
        m = self.file_re.match(text)
        if m:
            self.proto = m.group("proto")
            self.path = os.path.normpath(m.group("path"))
            return True
        m = self.scheme_re.match(text)
        if m and m.group("scheme") in self.BORGSTORE_SCHEMES:
            # borgstore parses/validates these; we only detect the scheme and pass the raw
            # URL (self.processed) through to it - no fields are extracted here.
            self.proto = m.group("scheme")
            return True
        m = self.local_re.match(text)
        if m:
            self.proto = "file"
            path = m.group("path")
            self.path = slashify(os.path.abspath(path)) if is_win32 else os.path.abspath(path)
            return True
        return False

    def __str__(self):
        items = [
            "proto=%r" % self.proto,
            "user=%r" % self.user,
            "pass=%r" % ("REDACTED" if self._pass else None),
            "host=%r" % self.host,
            "port=%r" % self.port,
            "path=%r" % self.path,
        ]
        return ", ".join(items)

    def __repr__(self):
        return "Location(%s)" % self

    @property
    def host(self):
        # strip square brackets used for IPv6 addrs
        if self._host is not None:
            return self._host.lstrip("[").rstrip("]")

    def canonical_path(self):
        if self.proto == "file":
            return normalize_local_path(self.path)
        if self.proto in ("rest", "ssh"):
            return (
                f"{self.proto}://"
                f"{(self.user + '@') if self.user else ''}"
                f"{self._host if self._host else ''}"
                f"{(':' + str(self.port)) if self.port else ''}/"
                f"{self.path}"
            )
        if self.proto in self.BORGSTORE_SCHEMES:
            # borgstore-handled locations: use the raw (processed) URL as given, but strip any
            # embedded credentials so we never write secrets to the security state file or logs.
            return _redact_url_credentials(self.processed)
        raise NotImplementedError(self.proto)

    def with_timestamp(self, timestamp):
        # note: this only affects the repository URL/path, not the archive name!
        return Location(
            self.raw,
            overrides={"now": DatetimeWrapper(timestamp), "utcnow": DatetimeWrapper(timestamp.astimezone(UTC))},
        )


def location_validator(proto=None, other=False):
    def validator(text):
        try:
            loc = Location(text, other=other)
        except ValueError as err:
            raise ArgumentTypeError(str(err)) from None
        if proto is not None and loc.proto != proto:
            if proto == "file":
                raise ArgumentTypeError('"%s": Repository must be local' % text)
            else:
                raise ArgumentTypeError('"%s": Repository must be remote' % text)
        return loc

    return validator


# Register types with jsonargparse so they can be represented in config files
# (e.g. for --print_config). Two things are needed:
# 1. A YAML representer so yaml.safe_dump can serialize Location objects to strings.
# 2. A jsonargparse register_type so it knows how to deserialize strings back to Location.

yaml.SafeDumper.add_representer(Location, lambda dumper, loc: dumper.represent_str(loc.raw or ""))
register_type(Location, serializer=lambda loc: loc.raw or "")

yaml.SafeDumper.add_representer(CompressionSpec, lambda dumper, cs: dumper.represent_str(str(cs)))
register_type(CompressionSpec)


def relative_time_marker_validator(text: str):
    time_marker_regex = r"^\d+[ymwdHMS]$"
    match = re.compile(time_marker_regex).search(text)
    if not match:
        raise ArgumentTypeError(f"Invalid relative time marker used: {text}, choose from y, m, w, d, H, M, S")
    else:
        return text


def text_validator(*, name, max_length, min_length=0, invalid_ctrl_chars="\0", invalid_chars="", no_blanks=False):
    def validator(text):
        assert isinstance(text, str)
        if len(text) < min_length:
            raise ArgumentTypeError(f'Invalid {name}: "{text}" [length < {min_length}]')
        if len(text) > max_length:
            raise ArgumentTypeError(f'Invalid {name}: "{text}" [length > {max_length}]')
        if invalid_ctrl_chars and re.search(f"[{re.escape(invalid_ctrl_chars)}]", text):
            raise ArgumentTypeError(f'Invalid {name}: "{text}" [invalid control chars detected]')
        if invalid_chars and re.search(f"[{re.escape(invalid_chars)}]", text):
            raise ArgumentTypeError(f'Invalid {name}: "{text}" [invalid chars detected matching "{invalid_chars}"]')
        if no_blanks and (text.startswith(" ") or text.endswith(" ")):
            raise ArgumentTypeError(f'Invalid {name}: "{text}" [leading or trailing blanks detected]')
        try:
            text.encode("utf-8", errors="strict")
        except UnicodeEncodeError:
            # looks like text contains surrogate-escapes
            raise ArgumentTypeError(f'Invalid {name}: "{text}" [contains non-unicode characters]')
        return text

    return validator


comment_validator = text_validator(name="comment", max_length=10000)
validate_tag_text = text_validator(name="tag", min_length=1, max_length=10, invalid_chars=" ,$")


def tag_validator(text):
    validated_text = validate_tag_text(text)
    if validated_text.startswith("@") and validated_text not in SPECIAL_TAGS:
        raise ArgumentTypeError("Unknown special tags given.")
    return validated_text


def archivename_validator(text):
    # we make sure that the archive name can be used as directory name (for borg mount)
    MAX_PATH = 260  # Windows default. Since Win10, there is a registry setting LongPathsEnabled to get more.
    MAX_DIRNAME = MAX_PATH - len("12345678.123")
    SAFETY_MARGIN = 48  # borgfs path: mountpoint / archivename / dir / dir / ... / file
    MAX_ARCHIVENAME = MAX_DIRNAME - SAFETY_MARGIN
    invalid_ctrl_chars = "".join(chr(i) for i in range(32))
    # note: ":" is also an invalid path char on windows, but we can not blacklist it,
    # because e.g. our {now} placeholder creates ISO-8601 like output like 2022-12-10T20:47:42 .
    invalid_chars = r"/" + r"\"<|>?*"  # posix + windows
    validate_text = text_validator(
        name="archive name",
        min_length=1,
        max_length=MAX_ARCHIVENAME,
        invalid_ctrl_chars=invalid_ctrl_chars,
        invalid_chars=invalid_chars,
        no_blanks=True,
    )
    return validate_text(text)


class BaseFormatter(metaclass=abc.ABCMeta):
    format: str
    static_data: dict[str, Any]
    FIXED_KEYS: ClassVar[dict[str, str]] = {
        # Formatting aids
        "LF": "\n",
        "SPACE": " ",
        "TAB": "\t",
        "CR": "\r",
        "NUL": "\0",
        "NEWLINE": "\n",
        "NL": "\n",  # \n is automatically converted to os.linesep on write
    }
    KEY_DESCRIPTIONS: ClassVar[dict[str, str]] = {
        "NEWLINE": "OS dependent line separator",
        "NL": "alias of NEWLINE",
        "NUL": "NUL character for creating print0 / xargs -0 like output",
        "SPACE": "space character",
        "TAB": "tab character",
        "CR": "carriage return character",
        "LF": "line feed character",
    }
    KEY_GROUPS: ClassVar[tuple[tuple[str, ...], ...]] = (("NEWLINE", "NL", "NUL", "SPACE", "TAB", "CR", "LF"),)

    def __init__(self, format: str, static: dict[str, Any]) -> None:
        self.validate_format(format)
        self.format = partial_format(format, static)
        self.static_data = static

    @abc.abstractmethod
    def get_item_data(self, item, jsonline=False) -> dict:
        raise NotImplementedError

    def format_item(self, item, jsonline=False, sort=False):
        data = self.get_item_data(item, jsonline)
        return (
            f"{json.dumps(data, cls=BorgJsonEncoder, sort_keys=sort)}\n" if jsonline else self.format.format_map(data)
        )

    @classmethod
    def keys_help(cls):
        help = []
        keys: set[str] = set()
        keys.update(cls.KEY_DESCRIPTIONS.keys())
        keys.update(key for group in cls.KEY_GROUPS for key in group)

        for group in cls.KEY_GROUPS:
            for key in group:
                keys.remove(key)
                text = "- " + key
                if key in cls.KEY_DESCRIPTIONS:
                    text += ": " + cls.KEY_DESCRIPTIONS[key]
                help.append(text)
            help.append("")
        assert not keys, str(keys)
        return "\n".join(help)

    @classmethod
    def known_keys(cls) -> set[str]:
        """the keys this formatter supports in a format string"""
        # some keys are only in KEY_GROUPS (e.g. the hash keys of ItemFormatter), thus we look at both.
        keys = set(cls.KEY_DESCRIPTIONS)
        keys.update(key for group in cls.KEY_GROUPS for key in group)
        keys.update(cls.FIXED_KEYS)
        return keys

    @classmethod
    def validate_format(cls, format):
        """raise a CommandError if format is malformed or uses keys this formatter does not know"""
        try:
            format_keys = {key for _, key, _, _ in Formatter().parse(format) if key}
        except ValueError as err:
            raise CommandError(f"Invalid format string: {err}")
        unknown_keys = format_keys - cls.known_keys()
        if unknown_keys:
            raise CommandError(f"Invalid format keys: {', '.join(sorted(unknown_keys))}")


class ArchiveFormatter(BaseFormatter):
    KEY_DESCRIPTIONS = {
        "archive": "archive name",
        "name": 'alias of "archive"',
        "comment": "archive comment",
        "tags": "archive tags",
        "time": "nominal time of the archive",
        "start": "start time of the archive operation",
        "end": "end time of the archive operation",
        "command_line": "command line which was used to create the archive",
        "id": "internal ID of the archive",
        "hostname": "hostname of host on which this archive was created",
        "username": "username of user who created this archive",
        "size": "size of this archive (data plus metadata, not considering compression and deduplication)",
        "nfiles": "count of files in this archive",
    }
    KEY_GROUPS = (
        ("archive", "name", "comment", "id", "tags"),
        ("time", "start", "end", "command_line"),
        ("hostname", "username"),
        ("size", "nfiles"),
    )

    def __init__(self, format, repository, manifest, key, *, deleted=False):
        static_data = {} | self.FIXED_KEYS  # here could be stuff on repo level, above archive level
        super().__init__(format, static_data)
        self.repository = repository
        self.manifest = manifest
        self.key = key
        self.name = None
        self.id = None
        self._archive = None
        self.deleted = deleted  # True if we want to deal with deleted archives.
        self.format_keys = {f[1] for f in Formatter().parse(format)}
        self.call_keys = {
            "hostname": partial(self.get_meta, "hostname", ""),
            "username": partial(self.get_meta, "username", ""),
            "comment": partial(self.get_meta, "comment", ""),
            "command_line": partial(self.get_meta, "command_line", ""),
            "size": partial(self.get_meta, "size", 0),
            "nfiles": partial(self.get_meta, "nfiles", 0),
            "start": self.get_ts_start,
            "end": self.get_ts_end,
            "tags": self.get_tags,
        }
        self.used_call_keys = set(self.call_keys) & self.format_keys

    def get_item_data(self, archive_info, jsonline=False):
        self.name = archive_info.name
        self.id = archive_info.id
        item_data = {}
        item_data |= {} if jsonline else self.static_data
        item_data |= {
            "name": archive_info.name,
            "archive": archive_info.name,
            "id": bin_to_hex(archive_info.id),
            "time": self.format_time(archive_info.ts),
        }
        for key in self.used_call_keys:
            item_data[key] = self.call_keys[key]()

        # Note: name and comment are validated, should never contain surrogate escapes.
        # But unsure whether hostname, username, command_line could contain surrogate escapes, play safe:
        for key in "hostname", "username", "command_line":
            if key in item_data:
                item_data |= text_to_json(key, item_data[key])
        return item_data

    @property
    def archive(self):
        """lazy load / update loaded archive"""
        if self._archive is None or self._archive.id != self.id:
            from ..archive import Archive

            self._archive = Archive(self.manifest, self.id, deleted=self.deleted)
        return self._archive

    def get_meta(self, key, default=None):
        return self.archive.metadata.get(key, default)

    def get_ts_start(self):
        return self.format_time(self.archive.ts_start)

    def get_ts_end(self):
        return self.format_time(self.archive.ts_end)

    def format_time(self, ts):
        return OutputTimestamp(ts)

    def get_tags(self):
        return ",".join(sorted(self.archive.tags))


class ItemFormatter(BaseFormatter):
    hash_algorithms = HASH_ALGORITHMS  # the same algos that "borg create --digests" can compute
    KEY_DESCRIPTIONS = {
        "type": "file type (file, dir, symlink, ...)",
        "mode": "file mode (as in stat)",
        "uid": "user id of file owner",
        "gid": "group id of file owner",
        "user": "user name of file owner",
        "group": "group name of file owner",
        "path": "file path",
        "target": "link target for symlinks",
        "hlid": "hard link identity (same if hardlinking same fs object)",
        "inode": "inode number",
        "flags": "file flags",
        "extra": 'prepends {target} with " -> " for soft links and " link to " for hard links',
        "size": "file size",
        "num_chunks": "number of chunks in this file",
        "mtime": "file modification time",
        "ctime": "file change time",
        "atime": "file access time",
        "isomtime": "file modification time (ISO 8601 format)",
        "isoctime": "file change time (ISO 8601 format)",
        "isoatime": "file access time (ISO 8601 format)",
        "fingerprint": "Fingerprint of the file content (may have false negatives), format: H(conditions)-H(chunk_ids)",
        "archiveid": "internal ID of the archive",
        "archivename": "name of the archive",
    }
    KEY_GROUPS = (
        ("type", "mode", "uid", "gid", "user", "group", "path", "target", "hlid", "inode", "flags"),
        ("size", "num_chunks"),
        ("mtime", "ctime", "atime", "isomtime", "isoctime", "isoatime"),
        tuple(["fingerprint"] + sorted(hash_algorithms)),
        ("archiveid", "archivename", "extra"),
    )

    KEYS_REQUIRING_CACHE = ()

    @classmethod
    def format_needs_cache(cls, format):
        format_keys = {f[1] for f in Formatter().parse(format)}
        return any(key in cls.KEYS_REQUIRING_CACHE for key in format_keys)

    def __init__(self, archive, format):
        static_data = {"archivename": archive.name, "archiveid": archive.fpr} | self.FIXED_KEYS
        super().__init__(format, static_data)
        self.archive = archive
        # track which keys were requested in the format string
        self.format_keys = {f[1] for f in Formatter().parse(format)}

        # we want a hash over the conditions that influence the chunk ID list for a given file content:
        # - the id algorithm and key
        # - the chunker seed (if any - buzhash64 derives seed from id_key)
        # - the chunker params
        key = archive.key
        conditions = f"{key.TYPE_STR!r}{key.id_key!r}{key.chunk_seed!r}{archive.metadata.get('chunker_params')!r}"
        self.conditions_hash = sha256(conditions.encode()).hexdigest()

        self.call_keys = {
            "size": self.calculate_size,
            "num_chunks": self.calculate_num_chunks,
            "isomtime": partial(self.format_iso_time, "mtime"),
            "isoctime": partial(self.format_iso_time, "ctime"),
            "isoatime": partial(self.format_iso_time, "atime"),
            "mtime": partial(self.format_time, "mtime"),
            "ctime": partial(self.format_time, "ctime"),
            "atime": partial(self.format_time, "atime"),
            "fingerprint": self.calculate_fingerprint,
        }
        for hash_function in self.hash_algorithms:
            self.call_keys[hash_function] = partial(self.hash_item, hash_function)
        self.used_call_keys = set(self.call_keys) & self.format_keys

    def get_item_data(self, item, jsonline=False):
        item_data = {}
        item_data |= {} if jsonline else self.static_data

        item_data |= text_to_json("path", item.path)
        target = item.get("target", "")
        item_data |= text_to_json("target", target)
        if not jsonline:
            item_data["extra"] = "" if not target else f" -> {item_data['target']}"

        hlid = item.get("hlid")
        hlid = bin_to_hex(hlid) if hlid else ""
        item_data["hlid"] = hlid

        mode = stat.filemode(item.mode)
        item_type = mode[0]
        item_data["type"] = item_type
        item_data["mode"] = mode

        item_data["uid"] = item.get("uid")  # int or None
        item_data["gid"] = item.get("gid")  # int or None
        item_data |= text_to_json("user", item.get("user", str(item_data["uid"])))
        item_data |= text_to_json("group", item.get("group", str(item_data["gid"])))

        item_data["flags"] = item.get("bsdflags")  # int if flags known, else (if flags unknown) None
        # inode number from source filesystem (may be absent on some platforms)
        item_data["inode"] = item.get("inode")
        for key in self.used_call_keys:
            item_data[key] = self.call_keys[key](item)
        # When producing JSON lines, include selected static archive-level keys if they were
        # requested via --format. This mirrors text output behavior and fixes #9095.
        if jsonline:
            # Include selected static archive-level keys when requested via --format.
            # Keep implementation style aligned with 1.4-maint.
            for k in ("archivename", "archiveid"):
                if k in self.format_keys:
                    item_data[k] = self.static_data[k]
        return item_data

    def calculate_num_chunks(self, item):
        return len(item.get("chunks", []))

    def calculate_size(self, item):
        # note: 0 for items without a chunks list, e.g. directories, symlinks, devices, fifos.
        # borg2 hard links all have their own chunks list, so they get their real size.
        return item.get_size()

    def calculate_fingerprint(self, item):
        # calculate a very fast file contents fingerprint
        chunks = item.get("chunks")
        if chunks is None:
            return ""
        chunks_hash = sha256(b"".join(c.id for c in chunks)).hexdigest()
        # we do not encounter many different conditions hashes, so the collision probability is low.
        # thus, we can keep it short and only return 64 bits from the conditions hash.
        return f"{self.conditions_hash[:16]}-{chunks_hash}"

    def hash_item(self, hash_function, item):
        if "chunks" not in item:
            return ""
        digest = item.get("digests", {}).get(hash_function)
        if digest is not None:
            # borg create computed this over the file content, so we do not need to read it again.
            return bin_to_hex(digest)
        if hash_function == "blake3":
            hash = blake3()
        else:
            hash = hashlib.new(hash_function)
        for data in self.archive.pipeline.fetch_many(item.chunks, ro_type=ROBJ_FILE_STREAM):
            hash.update(data)
        return hash.hexdigest()

    def format_time(self, key, item):
        return OutputTimestamp(safe_timestamp(item.get(key) or item.mtime))

    def format_iso_time(self, key, item):
        return self.format_time(key, item).isoformat()


class DiffFormatter(BaseFormatter):
    KEY_DESCRIPTIONS = {
        "path": "archived file path",
        "change": "all available changes",
        "content": "file content change",
        "mode": "file mode change",
        "type": "file type change",
        "owner": "file owner (user/group) change",
        "user": "file user change",
        "group": "file group change",
        "link": "file link change",
        "directory": "file directory change",
        "blkdev": "file block device change",
        "chrdev": "file character device change",
        "fifo": "file fifo change",
        "mtime": "file modification time change",
        "ctime": "file change time change",
        "isomtime": "file modification time change (ISO 8601)",
        "isoctime": "file creation time change (ISO 8601)",
    }
    KEY_GROUPS = (
        ("path", "change"),
        ("content", "mode", "type", "owner", "group", "user"),
        ("link", "directory", "blkdev", "chrdev", "fifo"),
        ("mtime", "ctime", "isomtime", "isoctime"),
    )
    METADATA = ("mode", "type", "owner", "group", "user", "mtime", "ctime")

    def __init__(self, format, content_only=False):
        static_data = {} | self.FIXED_KEYS
        super().__init__(format or "{content}{link}{directory}{blkdev}{chrdev}{fifo} {path}{NL}", static_data)
        self.content_only = content_only
        self.format_keys = {f[1] for f in Formatter().parse(format)}
        self.call_keys = {
            "content": self.format_content,
            "mode": self.format_mode,
            "type": partial(self.format_mode, filetype=True),
            "owner": partial(self.format_owner),
            "group": partial(self.format_owner, spec="group"),
            "user": partial(self.format_owner, spec="user"),
            "link": partial(self.format_other, "link"),
            "directory": partial(self.format_other, "directory"),
            "blkdev": partial(self.format_other, "blkdev"),
            "chrdev": partial(self.format_other, "chrdev"),
            "fifo": partial(self.format_other, "fifo"),
            "mtime": partial(self.format_time, "mtime"),
            "ctime": partial(self.format_time, "ctime"),
            "isomtime": partial(self.format_iso_time, "mtime"),
            "isoctime": partial(self.format_iso_time, "ctime"),
        }
        self.used_call_keys = set(self.call_keys) & self.format_keys
        if self.content_only:
            self.used_call_keys -= set(self.METADATA)

    def get_item_data(self, item: "ItemDiff", jsonline=False) -> dict:
        diff_data = {}
        for key in self.used_call_keys:
            diff_data[key] = self.call_keys[key](item)

        change = []
        for key in self.call_keys:
            if key in ("isomtime", "isoctime"):
                continue
            if self.content_only and key in self.METADATA:
                continue
            change.append(self.call_keys[key](item))
        diff_data["change"] = " ".join([v for v in change if v])
        diff_data["path"] = item.path
        diff_data |= {} if jsonline else self.static_data
        return diff_data

    def format_other(self, key, diff: "ItemDiff"):
        change = diff.changes().get(key)
        return f"{change.diff_type}".ljust(27) if change else ""  # 27 is the length of the content change

    def format_mode(self, diff: "ItemDiff", filetype=False):
        change = diff.type() if filetype else diff.mode()
        return f"[{change.diff_data['item1']} -> {change.diff_data['item2']}]" if change else ""

    def format_owner(self, diff: "ItemDiff", spec: Literal["owner", "user", "group"] = "owner"):
        if spec == "user":
            change = diff.user()
            return f"[{change.diff_data['item1']} -> {change.diff_data['item2']}]" if change else ""
        if spec == "group":
            change = diff.group()
            return f"[{change.diff_data['item1']} -> {change.diff_data['item2']}]" if change else ""
        if spec != "owner":
            raise ValueError(f"Invalid owner spec: {spec}")
        change = diff.owner()
        if change:
            return "[{}:{} -> {}:{}]".format(
                change.diff_data["item1"][0],
                change.diff_data["item1"][1],
                change.diff_data["item2"][0],
                change.diff_data["item2"][1],
            )
        return ""

    def format_content(self, diff: "ItemDiff"):
        change = diff.content()
        if change:
            if change.diff_type == "added":
                return "{}: {:>20}".format(change.diff_type, format_file_size(change.diff_data["added"]))
            if change.diff_type == "removed":
                return "{}: {:>18}".format(change.diff_type, format_file_size(change.diff_data["removed"]))
            if "added" not in change.diff_data and "removed" not in change.diff_data:
                return "modified:  (can't get size)"
            return "{}: {:>8} {:>8}".format(
                change.diff_type,
                format_file_size(change.diff_data["added"], precision=1, sign=True),
                format_file_size(-change.diff_data["removed"], precision=1, sign=True),
            )
        return ""

    def format_time(self, key, diff: "ItemDiff"):
        change = diff.changes().get(key)
        return f"[{key}: {change.diff_data['item1']} -> {change.diff_data['item2']}]" if change else ""

    def format_iso_time(self, key, diff: "ItemDiff"):
        change = diff.changes().get(key)
        return (
            f"[{key}: {change.diff_data['item1'].isoformat()} -> {change.diff_data['item2'].isoformat()}]"
            if change
            else ""
        )


def file_status(mode):
    if stat.S_ISREG(mode):
        return "A"
    elif stat.S_ISDIR(mode):
        return "d"
    elif stat.S_ISBLK(mode):
        return "b"
    elif stat.S_ISCHR(mode):
        return "c"
    elif stat.S_ISLNK(mode):
        return "s"
    elif stat.S_ISFIFO(mode):
        return "f"
    return "?"


def clean_lines(lines, lstrip=None, rstrip=None, remove_empty=True, remove_comments=True):
    """
    clean lines (usually read from a config file):

    1. strip whitespace (left and right), 2. remove empty lines, 3. remove comments.

    note: only "pure comment lines" are supported, no support for "trailing comments".

    :param lines: input line iterator (e.g. list or open text file) that gives unclean input lines
    :param lstrip: lstrip call arguments or False, if lstripping is not desired
    :param rstrip: rstrip call arguments or False, if rstripping is not desired
    :param remove_comments: remove comment lines (lines starting with "#")
    :param remove_empty: remove empty lines
    :return: yields processed lines
    """
    for line in lines:
        if lstrip is not False:
            line = line.lstrip(lstrip)
        if rstrip is not False:
            line = line.rstrip(rstrip)
        if remove_empty and not line:
            continue
        if remove_comments and line.startswith("#"):
            continue
        yield line


def swidth_slice(string, max_width):
    """
    Return a slice of *max_width* cells from *string*.

    Negative *max_width* means from the end of string.

    *max_width* is in units of character cells (or "columns").
    Latin characters are usually one cell wide, many CJK characters are two cells wide.
    """
    from ..platform import swidth

    reverse = max_width < 0
    max_width = abs(max_width)
    if reverse:
        string = reversed(string)
    current_swidth = 0
    result = []
    for character in string:
        current_swidth += swidth(character)
        if current_swidth > max_width:
            break
        result.append(character)
    if reverse:
        result.reverse()
    return "".join(result)


def ellipsis_truncate(msg, space):
    """
    shorten a long string by adding ellipsis between it and return it, example:
    this_is_a_very_long_string -------> this_is..._string
    """
    from ..platform import swidth

    ellipsis_width = swidth("...")
    msg_width = swidth(msg)
    if space < 8:
        # if there is very little space, just show ...
        return "..." + " " * (space - ellipsis_width)
    if space < ellipsis_width + msg_width:
        return f"{swidth_slice(msg, space // 2 - ellipsis_width)}...{swidth_slice(msg, -space // 2)}"
    return msg + " " * (space - msg_width)


class BorgJsonEncoder(json.JSONEncoder):
    def default(self, o):
        from ..legacy.repository import LegacyRepository
        from ..repository import Repository
        from ..legacy.remote import LegacyRemoteRepository
        from ..archive import Archive
        from ..cache import AdHocWithFilesCache

        if isinstance(o, (LegacyRepository, LegacyRemoteRepository)) or isinstance(o, Repository):
            return {"id": bin_to_hex(o.id), "location": o._location.canonical_path()}
        if isinstance(o, Archive):
            return o.info()
        if isinstance(o, (AdHocWithFilesCache,)):
            return {"path": o.path}
        if isinstance(o, Path):
            return str(o)
        if callable(getattr(o, "to_json", None)):
            return o.to_json()
        return super().default(o)


def basic_json_data(manifest, *, cache=None, extra=None):
    key = manifest.key
    data = extra or {}
    data |= {
        "repository": BorgJsonEncoder().default(manifest.repository),
        "encryption": {"encryption": key.ENC_NAME, "id_hash": key.IDHASH_NAME},
    }
    data["repository"]["last_modified"] = OutputTimestamp(manifest.last_timestamp)
    if getattr(key, "storage", None) == KeyBlobStorage.KEYFILE:
        data["encryption"]["keyfile"] = key.find_key()
    if cache:
        data["cache"] = cache
    return data


def _json_indent_from_env():
    """Parse BORG_JSON_INDENT env var for json.dumps indent parameter."""
    value = os.environ.get("BORG_JSON_INDENT")
    if value is None:
        return 4
    if value.lower() == "none":
        return None
    if value == "":
        return ""
    try:
        return int(value)
    except ValueError:
        return value


def json_dump(obj):
    """Dump using BorgJSONEncoder."""
    indent = _json_indent_from_env()
    return json.dumps(obj, sort_keys=True, indent=indent, cls=BorgJsonEncoder)


def json_print(obj):
    print(json_dump(obj))


def prepare_dump_dict(d):
    def decode_bytes(value):
        # this should somehow be reversible later, but usual strings should
        # look nice and chunk ids should mostly show in hex. Use a special
        # inband signaling character (ASCII DEL) to distinguish between
        # decoded and hex mode.
        if not value.startswith(b"\x7f"):
            try:
                value = value.decode()
                return value
            except UnicodeDecodeError:
                pass
        return "\u007f" + bin_to_hex(value)

    def decode_tuple(t):
        res = []
        for value in t:
            if isinstance(value, dict):
                value = decode(value)
            elif isinstance(value, tuple) or isinstance(value, list):
                value = decode_tuple(value)
            elif isinstance(value, bytes):
                value = decode_bytes(value)
            res.append(value)
        return res

    def decode(d):
        res = {}
        for key, value in d.items():
            if isinstance(value, dict):
                value = decode(value)
            elif isinstance(value, (tuple, list)):
                value = decode_tuple(value)
            elif isinstance(value, bytes):
                value = decode_bytes(value)
            elif isinstance(value, Timestamp):
                value = value.to_unix_nano()
            if isinstance(key, bytes):
                key = key.decode()
            res[key] = value
        return res

    return decode(d)


class Highlander(Action):
    """make sure some option is only given once"""

    def __init__(self, *args, **kwargs):
        self.__called = False
        super().__init__(*args, **kwargs)

    def __call__(self, parser, namespace, values, option_string=None):
        if self.__called:
            raise ArgumentError(self, "There can be only one.")
        self.__called = True
        setattr(namespace, self.dest, values)


class MakePathSafeAction(Highlander):
    def __call__(self, parser, namespace, path, option_string=None):
        try:
            sanitized_path = make_path_safe(path)
        except ValueError as e:
            raise ArgumentError(self, e)
        if sanitized_path == ".":
            raise ArgumentError(self, f"{path!r} is not a valid file name")
        setattr(namespace, self.dest, sanitized_path)
