/***********************************************************************************************************************************
Archive Get Command
***********************************************************************************************************************************/
#include <build.h>

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <unistd.h>

#include "command/archive/common.h"
#include "command/archive/get/file.h"
#include "command/archive/get/get.h"
#include "command/archive/get/protocol.h"
#include "command/command.h"
#include "command/lock.h"
#include "common/debug.h"
#include "common/log.h"
#include "common/memContext.h"
#include "common/regExp.h"
#include "common/wait.h"
#include "config/config.h"
#include "config/exec.h"
#include "config/load.h"
#include "info/infoArchive.h"
#include "postgres/interface.h"
#include "protocol/helper.h"
#include "protocol/parallel.h"
#include "storage/helper.h"
#include "storage/write.h"

/***********************************************************************************************************************************
Constants for log messages that are used multiple times to keep them consistent
***********************************************************************************************************************************/
#define FOUND_IN_ARCHIVE_MSG                                        "found %s in the archive"
#define FOUND_IN_REPO_ARCHIVE_MSG                                   "found %s in the %s: %s archive"
#define UNABLE_TO_FIND_IN_ARCHIVE_MSG                               "unable to find %s in the archive"
#define UNABLE_TO_FIND_VALID_REPO_MSG                               "unable to find a valid repository"
#define REPO_INVALID_OR_ERR_MSG                                     "some repositories were invalid or encountered errors"

/***********************************************************************************************************************************
Check for a list of archive files in the repository
***********************************************************************************************************************************/
typedef struct ArchiveFileMap
{
    const String *request;                                          // Archive file requested by archive_command
    List *actualList;                                               // Actual files in various repos/archiveIds
    StringList *warnList;                                           // Warnings that need to be reported by the async process
} ArchiveFileMap;

typedef struct ArchiveGetCheckResult
{
    List *archiveFileMapList;                                       // List of mapped archive files, i.e. found in the repo

    // Global error that affects all repos
    const ErrorType *errorType;                                     // Error type if there was an error
    const String *errorFile;                                        // Error file if there was an error
    const String *errorMessage;                                     // Error message if there was an error
    const StringList *warnList;                                     // Warnings that need to be reported by the async process
} ArchiveGetCheckResult;

// Helper to add an error to an error list and warn if the error is not already in the list
static void
archiveGetErrorAdd(
    StringList *const warnList, const bool log, const unsigned int repoIdx, const ErrorType *const type,
    const String *const message)
{
    const String *const warn = strNewFmt(
        "%s: [%s] %s", cfgOptionGroupName(cfgOptGrpRepo, repoIdx), errorTypeName(type), strZ(message));

    if (!strLstExists(warnList, warn))
    {
        if (log)
            LOG_WARN(strZ(warn));

        strLstAdd(warnList, warn);
    }
}

// Helper to find a single archive file in the repository using a cache to speed up the process and minimize storageListP() calls
typedef struct ArchiveGetFindCachePath
{
    const String *path;                                             // Cached path in the archiveId
    const StringList *fileList;                                     // List of files in the cache path
} ArchiveGetFindCachePath;

typedef struct ArchiveGetFindCacheArchive
{
    const String *archiveId;                                        // ArchiveId in the repo
    List *pathList;                                                 // List of paths cached for archiveId
} ArchiveGetFindCacheArchive;

typedef struct ArchiveGetFindCacheRepo
{
    unsigned int repoIdx;
    CipherType cipherType;                                          // Repo cipher type
    const String *cipherPassArchive;                                // Repo archive cipher pass
    List *archiveList;                                              // Cached list of archiveIds and associated paths
    StringList *warnList;                                           // Track repo warnings so each is only reported once
} ArchiveGetFindCacheRepo;

static bool
archiveGetFind(
    const String *const archiveFileRequest, ArchiveGetCheckResult *const getCheckResult, List *const cacheRepoList,
    const StringList *const warnList, const bool single)
{
    FUNCTION_LOG_BEGIN(logLevelDebug);
        FUNCTION_LOG_PARAM(STRING, archiveFileRequest);
        FUNCTION_LOG_PARAM_P(VOID, getCheckResult);
        FUNCTION_LOG_PARAM(LIST, cacheRepoList);
        FUNCTION_LOG_PARAM(STRING_LIST, warnList);
        FUNCTION_LOG_PARAM(BOOL, single);
    FUNCTION_LOG_END();

    ASSERT(archiveFileRequest != NULL);
    ASSERT(getCheckResult != NULL);
    ASSERT(cacheRepoList != NULL);

    bool result = false;

    MEM_CONTEXT_TEMP_BEGIN()
    {
        // Is the archive file a WAL segment?
        const bool isSegment = walIsSegment(archiveFileRequest);

        // Get the WAL segment path
        const String *const path = isSegment ? strSubN(archiveFileRequest, 0, 16) : NULL;

        // List to hold matches for the requested file
        List *const matchList = lstNewP(sizeof(ArchiveGetFile), .comparator = lstComparatorStr);

        // List of file level warnings
        StringList *const fileWarnList = strLstDup(warnList);

        // Errored repo total to track if all repos errored
        unsigned int repoErrorTotal = 0;

        // Check each repo
        for (unsigned int repoCacheIdx = 0; repoCacheIdx < lstSize(cacheRepoList); repoCacheIdx++)
        {
            const ArchiveGetFindCacheRepo *const cacheRepo = lstGet(cacheRepoList, repoCacheIdx);

            TRY_BEGIN()
            {
                // Check each archiveId
                for (unsigned int archiveCacheIdx = 0; archiveCacheIdx < lstSize(cacheRepo->archiveList); archiveCacheIdx++)
                {
                    const ArchiveGetFindCacheArchive *const cacheArchive = lstGet(cacheRepo->archiveList, archiveCacheIdx);

                    // If a WAL segment then search among the possible file names
                    if (isSegment)
                    {
                        StringList *segmentList;

                        // If a single file is requested then optimize by adding a restrictive expression to reduce bandwidth
                        if (single)
                        {
                            segmentList = storageListP(
                                storageRepoIdx(cacheRepo->repoIdx),
                                strNewFmt(STORAGE_REPO_ARCHIVE "/%s/%s", strZ(cacheArchive->archiveId), strZ(path)),
                                .expression = strNewFmt(
                                    "^%s%s-[0-f]{40}" COMPRESS_TYPE_REGEXP "{0,1}$", strZ(strSubN(archiveFileRequest, 0, 24)),
                                    walIsPartial(archiveFileRequest) ? WAL_SEGMENT_PARTIAL_EXT : ""));
                        }
                        // Else multiple files will be requested so cache list results
                        else
                        {
                            // Partial files cannot be in a list with multiple requests
                            ASSERT(!walIsPartial(archiveFileRequest));

                            // If the path does not exist in the cache then fetch it
                            const ArchiveGetFindCachePath *cachePath = lstFind(cacheArchive->pathList, &path);

                            if (cachePath == NULL)
                            {
                                MEM_CONTEXT_BEGIN(lstMemContext(cacheArchive->pathList))
                                {
                                    const ArchiveGetFindCachePath archiveGetFindCachePath =
                                    {
                                        .path = strDup(path),
                                        .fileList = storageListP(
                                            storageRepoIdx(cacheRepo->repoIdx),
                                            strNewFmt(STORAGE_REPO_ARCHIVE "/%s/%s", strZ(cacheArchive->archiveId), strZ(path)),
                                            .expression = strNewFmt(
                                                "^%s[0-F]{8}-[0-f]{40}" COMPRESS_TYPE_REGEXP "{0,1}$", strZ(path))),
                                    };

                                    cachePath = lstAdd(cacheArchive->pathList, &archiveGetFindCachePath);
                                }
                                MEM_CONTEXT_END();
                            }

                            // Get a list of all WAL segments that match
                            segmentList = strLstNew();

                            for (unsigned int fileIdx = 0; fileIdx < strLstSize(cachePath->fileList); fileIdx++)
                            {
                                if (strBeginsWith(strLstGet(cachePath->fileList, fileIdx), archiveFileRequest))
                                    strLstAdd(segmentList, strLstGet(cachePath->fileList, fileIdx));
                            }
                        }

                        // Add segments to match list
                        for (unsigned int segmentIdx = 0; segmentIdx < strLstSize(segmentList); segmentIdx++)
                        {
                            MEM_CONTEXT_BEGIN(lstMemContext(getCheckResult->archiveFileMapList))
                            {
                                const ArchiveGetFile archiveGetFile =
                                {
                                    .file = strNewFmt(
                                        "%s/%s/%s", strZ(cacheArchive->archiveId), strZ(path),
                                        strZ(strLstGet(segmentList, segmentIdx))),
                                    .repoIdx = cacheRepo->repoIdx,
                                    .archiveId = cacheArchive->archiveId,
                                    .cipherType = cacheRepo->cipherType,
                                    .cipherPassArchive = cacheRepo->cipherPassArchive,
                                };

                                lstAdd(matchList, &archiveGetFile);
                            }
                            MEM_CONTEXT_END();
                        }
                    }
                    // Else if not a WAL segment, see if it exists in the archiveId path
                    else if (
                        storageExistsP(
                            storageRepoIdx(cacheRepo->repoIdx),
                            strNewFmt(STORAGE_REPO_ARCHIVE "/%s/%s", strZ(cacheArchive->archiveId), strZ(archiveFileRequest))))
                    {
                        MEM_CONTEXT_BEGIN(lstMemContext(getCheckResult->archiveFileMapList))
                        {
                            const ArchiveGetFile archiveGetFile =
                            {
                                .file = strNewFmt("%s/%s", strZ(cacheArchive->archiveId), strZ(archiveFileRequest)),
                                .repoIdx = cacheRepo->repoIdx,
                                .archiveId = cacheArchive->archiveId,
                                .cipherType = cacheRepo->cipherType,
                                .cipherPassArchive = cacheRepo->cipherPassArchive,
                            };

                            lstAdd(matchList, &archiveGetFile);
                        }
                        MEM_CONTEXT_END();
                    }
                }
            }
            // Log errors as warnings and continue
            CATCH_ANY()
            {
                repoErrorTotal++;
                archiveGetErrorAdd(cacheRepo->warnList, true, cacheRepo->repoIdx, errorType(), STR(errorMessage()));
                archiveGetErrorAdd(fileWarnList, false, cacheRepo->repoIdx, errorType(), STR(errorMessage()));
            }
            TRY_END();
        }

        // If all repos errored out then set the global error since processing cannot continue past this segment
        ASSERT(repoErrorTotal <= lstSize(cacheRepoList));

        if (repoErrorTotal == lstSize(cacheRepoList))
        {
            ASSERT(!strLstEmpty(fileWarnList));

            MEM_CONTEXT_BEGIN(lstMemContext(getCheckResult->archiveFileMapList))
            {
                getCheckResult->errorType = &RepoInvalidError;
                getCheckResult->errorFile = strDup(archiveFileRequest);
                getCheckResult->errorMessage = strNewZ(UNABLE_TO_FIND_VALID_REPO_MSG);
                getCheckResult->warnList = strLstMove(fileWarnList, memContextCurrent());
            }
            MEM_CONTEXT_END();
        }
        // Else if a file was found
        else if (!lstEmpty(matchList))
        {
            bool error = false;

            // If a segment match list is > 1 then check for duplicates
            if (isSegment && lstSize(matchList) > 1)
            {
                // Count the number of unique hashes
                StringList *const hashList = strLstNew();

                for (unsigned int matchIdx = 0; matchIdx < lstSize(matchList); matchIdx++)
                    strLstAddIfMissing(hashList, strSubN(((ArchiveGetFile *)lstGet(matchList, matchIdx))->file, 25, 40));

                // If there is more than one unique hash then there are duplicates
                if (strLstSize(hashList) > 1)
                {
                    // Build list of duplicates
                    unsigned int repoKeyLast = 0;
                    String *const message = strNew();
                    bool first = true;

                    // Sort the matches so they are logged in a consistent order
                    lstSort(matchList, sortOrderAsc);

                    for (unsigned int matchIdx = 0; matchIdx < lstSize(matchList); matchIdx++)
                    {
                        const ArchiveGetFile *const file = lstGet(matchList, matchIdx);
                        const unsigned int repoKey = cfgOptionGroupIdxToKey(cfgOptGrpRepo, file->repoIdx);

                        if (repoKey != repoKeyLast)
                        {
                            strCatFmt(message, "\n%s:", cfgOptionGroupName(cfgOptGrpRepo, file->repoIdx));
                            repoKeyLast = repoKey;
                            first = true;
                        }

                        if (first)
                            first = false;
                        else
                            strCatChr(message, ',');

                        strCatFmt(message, " %s", strZ(file->file));
                    }

                    // Set as global error since processing cannot continue past this segment
                    MEM_CONTEXT_BEGIN(lstMemContext(getCheckResult->archiveFileMapList))
                    {
                        getCheckResult->errorType = &ArchiveDuplicateError;
                        getCheckResult->errorFile = strDup(archiveFileRequest);
                        getCheckResult->errorMessage = strNewFmt(
                            "duplicates found for WAL segment %s:%s\n"
                            "HINT: are multiple primaries archiving to this stanza?",
                            strZ(archiveFileRequest), strZ(message));
                        getCheckResult->warnList = strLstMove(fileWarnList, memContextCurrent());
                    }
                    MEM_CONTEXT_END();

                    error = true;
                }
            }

            // Files are valid so add them to the map
            if (!error)
            {
                MEM_CONTEXT_BEGIN(lstMemContext(getCheckResult->archiveFileMapList))
                {
                    const ArchiveFileMap map =
                    {
                        .request = strDup(archiveFileRequest),
                        .actualList = lstNewP(sizeof(ArchiveGetFile)),
                        .warnList = strLstMove(fileWarnList, memContextCurrent()),
                    };

                    for (unsigned int matchIdx = 0; matchIdx < lstSize(matchList); matchIdx++)
                        lstAdd(map.actualList, lstGet(matchList, matchIdx));

                    lstAdd(getCheckResult->archiveFileMapList, &map);
                }
                MEM_CONTEXT_END();

                result = true;
            }
        }
    }
    MEM_CONTEXT_TEMP_END();

    FUNCTION_LOG_RETURN(BOOL, result);
}

static ArchiveGetCheckResult
archiveGetCheck(const StringList *const archiveRequestList)
{
    FUNCTION_LOG_BEGIN(logLevelDebug);
        FUNCTION_LOG_PARAM(STRING_LIST, archiveRequestList);
    FUNCTION_LOG_END();

    FUNCTION_AUDIT_STRUCT();

    ASSERT(archiveRequestList != NULL);
    ASSERT(!strLstEmpty(archiveRequestList));

    ArchiveGetCheckResult result = {.archiveFileMapList = lstNewP(sizeof(ArchiveFileMap), .comparator = lstComparatorStr)};

    MEM_CONTEXT_TEMP_BEGIN()
    {
        // List of warnings
        StringList *const warnList = strLstNew();

        // Get pg_control info. Error if the invalid checkpoint written by restore is detected and backup_label is not present.
        const PgControl controlInfo = pgControlFromFile(storagePg(), cfgOptionStrNull(cfgOptPgVersionForce));

        if (controlInfo.checkpoint == PG_CONTROL_CHECKPOINT_INVALID && !storageExistsP(storagePg(), STRDEF(PG_FILE_BACKUPLABEL)))
        {
            THROW(
                FormatError,
                "pg_control from backup is not valid without backup_label\n"
                "HINT: was the backup_label file removed?");
        }

        // Build list of repos/archiveIds where WAL may be found
        List *const cacheRepoList = lstNewP(sizeof(ArchiveGetFindCacheRepo));

        for (unsigned int repoIdx = 0; repoIdx < cfgOptionGroupIdxTotal(cfgOptGrpRepo); repoIdx++)
        {
            // If a repo was specified then skip all other repos
            if (cfgOptionTest(cfgOptRepo) && cfgOptionUInt(cfgOptRepo) != cfgOptionGroupIdxToKey(cfgOptGrpRepo, repoIdx))
                continue;

            TRY_BEGIN()
            {
                // Get the repo storage in case it is remote and encryption settings need to be pulled down
                storageRepoIdx(repoIdx);

                ArchiveGetFindCacheRepo cacheRepo =
                {
                    .repoIdx = repoIdx,
                    .cipherType = cfgOptionIdxStrId(cfgOptRepoCipherType, repoIdx),
                    .archiveList = lstNewP(sizeof(ArchiveGetFindCacheArchive)),
                    .warnList = strLstNew(),
                };

                // Attempt to load the archive info file
                const InfoArchive *const info = infoArchiveLoadFile(
                    storageRepoIdx(repoIdx), INFO_ARCHIVE_PATH_FILE_STR, cacheRepo.cipherType,
                    cfgOptionIdxStrNull(cfgOptRepoCipherPass, repoIdx));

                // Copy cipher pass into the result list context once rather than making a copy per candidate file later
                MEM_CONTEXT_BEGIN(lstMemContext(result.archiveFileMapList))
                {
                    cacheRepo.cipherPassArchive = strDup(infoArchiveCipherPass(info));
                }
                MEM_CONTEXT_END();

                // Loop through pg history and determine which archiveIds to use
                const StringList *archivePathList = NULL;

                for (unsigned int pgIdx = 0; pgIdx < infoPgDataTotal(infoArchivePg(info)); pgIdx++)
                {
                    InfoPgData pgData = infoPgData(infoArchivePg(info), pgIdx);

                    // Only use the archive id if it matches the current cluster
                    if (pgData.systemId == controlInfo.systemId && pgData.version == controlInfo.version)
                    {
                        const String *const archiveId = infoPgArchiveId(infoArchivePg(info), pgIdx);
                        bool found = true;

                        // If the archiveId is in the past make sure the path exists
                        if (pgIdx != 0)
                        {
                            // Get list of archiveId paths in the archive path
                            if (archivePathList == NULL)
                                archivePathList = storageListP(storageRepoIdx(repoIdx), STORAGE_REPO_ARCHIVE_STR);

                            if (!strLstExists(archivePathList, archiveId))
                                found = false;
                        }

                        // If the archiveId is most recent or has files then add it
                        if (found)
                        {
                            ArchiveGetFindCacheArchive cacheArchive =
                            {
                                .pathList = lstNewP(sizeof(ArchiveGetFindCachePath), .comparator = lstComparatorStr),
                            };

                            // Copy archiveId into the result list context once rather than making a copy per candidate file later
                            MEM_CONTEXT_BEGIN(lstMemContext(result.archiveFileMapList))
                            {
                                cacheArchive.archiveId = strDup(archiveId);
                            }
                            MEM_CONTEXT_END();

                            lstAdd(cacheRepo.archiveList, &cacheArchive);
                        }
                    }
                }

                // Error if no archive id was found -- this indicates a mismatch with the current cluster
                if (lstEmpty(cacheRepo.archiveList))
                {
                    archiveGetErrorAdd(
                        warnList, true, repoIdx, &ArchiveMismatchError,
                        strNewFmt(
                            "unable to retrieve the archive id for database version '%s' and system-id '%" PRIu64 "'",
                            strZ(pgVersionToStr(controlInfo.version)), controlInfo.systemId));
                }
                // Else add repo to list
                else
                    lstAdd(cacheRepoList, &cacheRepo);
            }
            // Log errors as warnings and continue
            CATCH_ANY()
            {
                archiveGetErrorAdd(warnList, true, repoIdx, errorType(), STR(errorMessage()));
            }
            TRY_END();
        }

        // Error if there are no repos to check
        if (lstEmpty(cacheRepoList))
        {
            ASSERT(!strLstEmpty(warnList));

            // Set as global error since processing cannot continue past this segment
            MEM_CONTEXT_BEGIN(lstMemContext(result.archiveFileMapList))
            {
                result.errorType = &RepoInvalidError;
                result.errorMessage = strNewZ(UNABLE_TO_FIND_VALID_REPO_MSG);
                result.warnList = strLstMove(warnList, memContextCurrent());
            }
            MEM_CONTEXT_END();
        }
        else
        {
            // Any remaining errors will be reported as warnings since at least one repo is valid
            MEM_CONTEXT_BEGIN(lstMemContext(result.archiveFileMapList))
            {
                result.warnList = strLstMove(warnList, memContextCurrent());
            }
            MEM_CONTEXT_END();

            // Find files in the list
            for (unsigned int archiveRequestIdx = 0; archiveRequestIdx < strLstSize(archiveRequestList); archiveRequestIdx++)
            {
                if (!archiveGetFind(
                        strLstGet(archiveRequestList, archiveRequestIdx), &result, cacheRepoList, warnList,
                        strLstSize(archiveRequestList) == 1))
                {
                    break;
                }
            }

            // Sort the list to make searching for files faster
            lstSort(result.archiveFileMapList, sortOrderAsc);
        }
    }
    MEM_CONTEXT_TEMP_END();

    FUNCTION_LOG_RETURN_STRUCT(result);
}

/***********************************************************************************************************************************
Clean the queue and prepare a list of WAL segments that the async process should get
***********************************************************************************************************************************/
static StringList *
queueNeed(
    const String *const walSegment, const bool found, const uint64_t queueSize, const size_t walSegmentSize,
    const unsigned int pgVersion)
{
    FUNCTION_LOG_BEGIN(logLevelDebug);
        FUNCTION_LOG_PARAM(STRING, walSegment);
        FUNCTION_LOG_PARAM(BOOL, found);
        FUNCTION_LOG_PARAM(UINT64, queueSize);
        FUNCTION_LOG_PARAM(SIZE, walSegmentSize);
        FUNCTION_LOG_PARAM(UINT, pgVersion);
    FUNCTION_LOG_END();

    ASSERT(walSegment != NULL);

    StringList *const result = strLstNew();

    MEM_CONTEXT_TEMP_BEGIN()
    {
        // Determine the first WAL segment for the async process to get. If the WAL segment requested by PostgreSQL was not found
        // then use that. If the segment was found but the queue is not full then start with the next segment.
        const String *const walSegmentFirst =
            found ? walSegmentNext(walSegment, walSegmentSize, pgVersion) : walSegment;

        // Determine how many WAL segments should be in the queue. The queue total must be at least 2 or it doesn't make sense to
        // have async turned on at all.
        unsigned int walSegmentQueueTotal = (unsigned int)(queueSize / walSegmentSize);

        if (walSegmentQueueTotal < 2)
            walSegmentQueueTotal = 2;

        // Build the ideal queue -- the WAL segments we want in the queue after the async process has run
        const StringList *const idealQueue = strLstSort(
            walSegmentRange(walSegmentFirst, walSegmentSize, pgVersion, walSegmentQueueTotal), sortOrderAsc);

        // Get the list of files actually in the queue
        const StringList *const actualQueue = strLstSort(
            storageListP(storageSpool(), STORAGE_SPOOL_ARCHIVE_IN_STR, .errorOnMissing = true), sortOrderAsc);

        // Build a list of WAL segments that are being kept so we can later make a list of what is needed
        StringList *const keepQueue = strLstNew();

        for (unsigned int actualQueueIdx = 0; actualQueueIdx < strLstSize(actualQueue); actualQueueIdx++)
        {
            // Get file from actual queue
            const String *const file = strLstGet(actualQueue, actualQueueIdx);

            // Does this match a file we want to preserve?
            if (strLstExists(idealQueue, file))
            {
                strLstAdd(keepQueue, file);
            }
            // Else delete if it does not match an ok file for a WAL segment that has already been preserved. If an ok file exists
            // in addition to the segment then it contains warnings which need to be preserved.
            else if (
                !strEndsWithZ(file, STATUS_EXT_OK) ||
                !strLstExists(actualQueue, strSubN(file, 0, strSize(file) - STATUS_EXT_OK_SIZE)))
            {
                storageRemoveP(storageSpoolWrite(), strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s", strZ(file)), .errorOnMissing = true);
            }
        }

        // Generate a list of the WAL that are needed by removing kept WAL from the ideal queue
        strLstSort(keepQueue, sortOrderAsc);

        for (unsigned int idealQueueIdx = 0; idealQueueIdx < strLstSize(idealQueue); idealQueueIdx++)
        {
            if (!strLstExists(keepQueue, strLstGet(idealQueue, idealQueueIdx)))
                strLstAdd(result, strLstGet(idealQueue, idealQueueIdx));
        }
    }
    MEM_CONTEXT_TEMP_END();

    FUNCTION_LOG_RETURN(STRING_LIST, result);
}

/***********************************************************************************************************************************
Get a single WAL segment synchronously, i.e. copy it directly from the repository to the destination rather than using the async
spool queue. Returns 0 if the segment was found and copied, else 1. Throws on error.
***********************************************************************************************************************************/
static int
archiveGetSingle(const String *const walSegment, const String *const walDestination)
{
    FUNCTION_LOG_BEGIN(logLevelDebug);
        FUNCTION_LOG_PARAM(STRING, walSegment);
        FUNCTION_LOG_PARAM(STRING, walDestination);
    FUNCTION_LOG_END();

    ASSERT(walSegment != NULL);
    ASSERT(walDestination != NULL);

    int result = 1;

    MEM_CONTEXT_TEMP_BEGIN()
    {
        // Check for the archive file
        StringList *const archiveRequestList = strLstNew();
        strLstAdd(archiveRequestList, walSegment);

        const ArchiveGetCheckResult checkResult = archiveGetCheck(archiveRequestList);

        // If there was an error then throw it
        if (checkResult.errorType != NULL)
            THROW_CODE(errorTypeCode(checkResult.errorType), strZ(checkResult.errorMessage));

        // Get the archive file
        if (!lstEmpty(checkResult.archiveFileMapList))
        {
            // There can only be one file mapping since only one file was requested
            ASSERT(lstSize(checkResult.archiveFileMapList) == 1);
            const ArchiveFileMap *const fileMap = lstGet(checkResult.archiveFileMapList, 0);

            // Get the file
            const ArchiveGetFileResult fileResult = archiveGetFile(
                storageLocalWrite(), fileMap->request, fileMap->actualList, walDestination);

            // Output file warnings
            for (unsigned int warnIdx = 0; warnIdx < strLstSize(fileResult.warnList); warnIdx++)
                LOG_WARN(strZ(strLstGet(fileResult.warnList, warnIdx)));

            // If there was no error then the file existed
            const ArchiveGetFile *const file = lstGet(fileMap->actualList, fileResult.actualIdx);
            ASSERT(file != NULL);

            LOG_INFO_FMT(
                FOUND_IN_REPO_ARCHIVE_MSG, strZ(walSegment), cfgOptionGroupName(cfgOptGrpRepo, file->repoIdx),
                strZ(file->archiveId));

            result = 0;
        }
        // Else log that the file was not found
        else
            LOG_INFO_FMT(UNABLE_TO_FIND_IN_ARCHIVE_MSG, strZ(walSegment));
    }
    MEM_CONTEXT_TEMP_END();

    FUNCTION_LOG_RETURN(INT, result);
}

/**********************************************************************************************************************************/
FN_EXTERN int
cmdArchiveGet(void)
{
    FUNCTION_LOG_VOID(logLevelDebug);

    // PostgreSQL must be local
    pgIsLocalVerify();

    // Set the result assuming the archive file will not be found
    int result = 1;

    MEM_CONTEXT_TEMP_BEGIN()
    {
        // Check the parameters
        const StringList *const commandParam = cfgCommandParam();

        if (strLstSize(commandParam) != 2)
        {
            if (strLstEmpty(commandParam))
                THROW(ParamRequiredError, "WAL segment to get required");

            if (strLstSize(commandParam) == 1)
                THROW(ParamRequiredError, "path to copy WAL segment required");

            THROW(ParamInvalidError, "extra parameters found");
        }

        // Get the segment name
        const String *const walSegment = strBase(strLstGet(commandParam, 0));

        // Destination is wherever we were told to move the WAL segment
        const String *const walDestination =
            walPath(strLstGet(commandParam, 1), cfgOptionStr(cfgOptPgPath), STR(cfgCommandName()));

        // Async get can only be performed on WAL segments, history or other files must use synchronous mode
        if (cfgOptionBool(cfgOptArchiveAsync) && walIsSegment(walSegment))
        {
            bool first = true;                                          // Is this the first time the loop has run?
            bool found = false;                                         // Has the WAL segment been found yet?
            bool foundOk = false;                                       // Was an OK file found which confirms the file was missing?
            bool queueFull = false;                                     // Is the queue half or more full?
            bool forked = false;                                        // Has the async process been forked yet?
            bool syncGet = false;                                       // Was the segment resolved by a synchronous get?

            // Loop and wait for the WAL segment to be pushed
            Wait *const wait = waitNew(cfgOptionUInt64(cfgOptArchiveTimeout));

            do
            {
                // Check if the WAL segment is already in the queue
                found = storageExistsP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s", strZ(walSegment)));

                // Determine whether a missing WAL segment will be retried. Retrying is safer, but not retrying lets PostgreSQL
                // know that there are probably no more WAL segments in the archive which means it can switch to streaming.
                const bool missingRetry = first && cfgOptionBool(cfgOptArchiveMissingRetry);

                // Check for errors or missing files. For archive-get ok indicates that the process succeeded but there is no WAL
                // file to download, or that there was a warning. Do not error on the first run so the async process can be spawned
                // to correct any errors from a previous run. Do not warn on the first run if the segment was not found so the async
                // process can be spawned to check for the file again.
                if (archiveAsyncStatus(archiveModeGet, walSegment, !first, found || !missingRetry))
                {
                    storageRemoveP(
                        storageSpoolWrite(), strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s" STATUS_EXT_OK, strZ(walSegment)),
                        .errorOnMissing = true);

                    // Break if an ok file was found but no segment exists, which means the segment was missing. However, don't
                    // break if this is the first time through the loop since this means the ok file was written by an async process
                    // spawned by a prior archive-get execution, which means we should spawn the async process again to see if the
                    // file exists now. This also prevents spool files from a previous recovery interfering with the current
                    // recovery.
                    if (!found && !missingRetry)
                    {
                        foundOk = true;
                        break;
                    }
                }

                // If found then move the WAL segment to the destination directory
                if (found)
                {
                    // Source is the WAL segment in the spool queue
                    StorageRead *const source = storageNewReadP(
                        storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s", strZ(walSegment)));

                    // A move will be attempted but if the spool queue and the WAL path are on different file systems then a copy
                    // will be performed instead.
                    //
                    // It looks scary that we are disabling syncs and atomicity (in case we need to copy instead of move) but this
                    // is safe because if the system crashes Postgres will not try to reuse a restored WAL segment but will instead
                    // request it again using the restore_command. In the case of a move this hardly matters since path syncs are
                    // cheap but if a copy is required we could save a lot of writes.
                    StorageWrite *const destination = storageNewWriteP(
                        storageLocalWrite(), walDestination, .noCreatePath = true, .noSyncFile = true, .noSyncPath = true,
                        .noAtomic = true);

                    // Move (or copy if required) the file
                    storageMoveP(storageSpoolWrite(), source, destination);

                    // Return success
                    LOG_INFO_FMT(FOUND_IN_ARCHIVE_MSG " asynchronously", strZ(walSegment));
                    result = 0;

                    // Get a list of WAL segments left in the queue
                    const StringList *const queue = storageListP(
                        storageSpool(), STORAGE_SPOOL_ARCHIVE_IN_STR, .expression = WAL_SEGMENT_REGEXP_STR, .errorOnMissing = true);

                    if (!strLstEmpty(queue))
                    {
                        // Get size of the WAL segment
                        const uint64_t walSegmentSize = storageInfoP(storageLocal(), walDestination).size;

                        // Use WAL segment size to estimate queue size and determine if the async process should be launched
                        queueFull = strLstSize(queue) * walSegmentSize > cfgOptionUInt64(cfgOptArchiveGetQueueMax) / 2;
                    }
                }

                // If the WAL segment has not already been found then start the async process to get it. There's no point in forking
                // the async process off more than once so track that as well. Use an archive lock to prevent forking if the async
                // process was launched by another process.
                if (!forked && (!found || !queueFull) && cmdLockAcquireP(.returnOnNoLock = true))
                {
                    // Get control info
                    const PgControl pgControl = pgControlFromFile(storagePg(), cfgOptionStrNull(cfgOptPgVersionForce));

                    // Create the queue
                    storagePathCreateP(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN_STR);

                    // The async process should not output on the console at all
                    KeyValue *const optionReplace = kvNew();

                    kvPut(optionReplace, VARSTRDEF(CFGOPT_LOG_LEVEL_CONSOLE), VARSTRDEF("off"));
                    kvPut(optionReplace, VARSTRDEF(CFGOPT_LOG_LEVEL_STDERR), VARSTRDEF("off"));

                    // Generate command options
                    StringList *const commandExec = cfgExecParam(cfgCmdArchiveGet, cfgCmdRoleAsync, optionReplace, true, false);
                    strLstInsert(commandExec, 0, cfgBin());

                    // Clean the current queue using the list of WAL that we ideally want in the queue. queueNeed() will return the
                    // list of WAL needed to fill the queue and this will be passed to the async process.
                    const StringList *const queue = queueNeed(
                        walSegment, found, cfgOptionUInt64(cfgOptArchiveGetQueueMax), pgControl.walSegmentSize,
                        pgControl.version);

                    for (unsigned int queueIdx = 0; queueIdx < strLstSize(queue); queueIdx++)
                        strLstAdd(commandExec, strLstGet(queue, queueIdx));

                    // Clear errors for the current wal segment
                    archiveAsyncErrorClear(archiveModeGet, walSegment);

                    // Release the lock so the child process can acquire it
                    cmdLockReleaseP();

                    // Execute the async process
                    cmdAsyncExec(CFGCMD_ARCHIVE_GET, commandExec);

                    // Mark the async process as forked so it doesn't get forked again. A single run of the async process should be
                    // enough to do the job, running it again won't help anything.
                    forked = true;
                }

                // If the segment is not in the queue and our own async process is not holding the lock then the segment will not be
                // fetched by us: either a foreign async process holds the lock (spawned by a different main process, e.g. filling
                // the queue for the old timeline after a timeline switch) and will not fetch our segment, or no async process is
                // running because ours failed. Rather than wait for it and risk a timeout on a segment that actually exists --
                // which can end recovery prematurely -- fetch the segment synchronously. Skip the first run so an async process
                // spawned above (or by a prior invocation) gets a chance to take the lock first.
                if (!found && !first && !cmdLockOwn())
                {
                    result = archiveGetSingle(walSegment, walDestination);
                    found = result == 0;
                    syncGet = true;

                    break;
                }

                // Exit loop if WAL was found
                if (found)
                    break;

                // No longer the first run, so errors will be thrown and missing files will be reported
                first = false;
            }
            while (waitMore(wait));

            // If the WAL segment was not found and it was not resolved by a synchronous get (which does its own logging)
            if (!found && !syncGet)
            {
                // If no ok file was found then something may be wrong with the async process. It's better to throw an error here
                // than report not found for debugging purposes. Either way PostgreSQL will halt if it has not reached consistency.
                if (!foundOk)
                {
                    THROW_FMT(
                        ArchiveTimeoutError,
                        "unable to get WAL file '%s' from the archive asynchronously after %s second(s)\n"
                        "HINT: check '%s' for errors.",
                        strZ(walSegment),
                        strZ(
                            strNewDivP((uint64_t)cfgOptionInt64(cfgOptArchiveTimeout), MSEC_PER_SEC, .precision = 3, .trim = true)),
                        strZ(cfgLoadLogFileName(cfgCmdRoleAsync)));
                }
                // Else report that the WAL segment could not be found
                else
                    LOG_INFO_FMT(UNABLE_TO_FIND_IN_ARCHIVE_MSG " asynchronously", strZ(walSegment));
            }
        }
        // Else perform synchronous get
        else
            result = archiveGetSingle(walSegment, walDestination);
    }
    MEM_CONTEXT_TEMP_END();

    FUNCTION_LOG_RETURN(INT, result);
}

/**********************************************************************************************************************************/
typedef struct ArchiveGetAsyncData
{
    const List *const archiveFileMapList;                           // List of wal segments to process
    unsigned int archiveFileIdx;                                    // Current index in the list to be processed
} ArchiveGetAsyncData;

static ProtocolParallelJob *
archiveGetAsyncCallback(void *const data, const unsigned int clientIdx)
{
    FUNCTION_TEST_BEGIN();
        FUNCTION_TEST_PARAM_P(VOID, data);
        FUNCTION_TEST_PARAM(UINT, clientIdx);
    FUNCTION_TEST_END();

    ProtocolParallelJob *result = NULL;

    MEM_CONTEXT_TEMP_BEGIN()
    {
        // No special logic based on the client, we'll just get the next job
        (void)clientIdx;

        // Get a new job if there are any left
        ArchiveGetAsyncData *const jobData = data;

        if (jobData->archiveFileIdx < lstSize(jobData->archiveFileMapList))
        {
            const ArchiveFileMap *const archiveFileMap = lstGet(jobData->archiveFileMapList, jobData->archiveFileIdx);
            jobData->archiveFileIdx++;

            PackWrite *const param = protocolPackNew();

            pckWriteStrP(param, archiveFileMap->request);

            // Add actual files to get
            for (unsigned int actualIdx = 0; actualIdx < lstSize(archiveFileMap->actualList); actualIdx++)
            {
                const ArchiveGetFile *const actual = lstGet(archiveFileMap->actualList, actualIdx);

                pckWriteStrP(param, actual->file);
                pckWriteU32P(param, actual->repoIdx);
                pckWriteStrP(param, actual->archiveId);
                pckWriteU64P(param, actual->cipherType);
                pckWriteStrP(param, actual->cipherPassArchive);
            }

            MEM_CONTEXT_PRIOR_BEGIN()
            {
                result = protocolParallelJobNew(VARSTR(archiveFileMap->request), PROTOCOL_COMMAND_ARCHIVE_GET_FILE, param);
            }
            MEM_CONTEXT_PRIOR_END();
        }
    }
    MEM_CONTEXT_TEMP_END();

    FUNCTION_TEST_RETURN(PROTOCOL_PARALLEL_JOB, result);
}

FN_EXTERN void
cmdArchiveGetAsync(void)
{
    FUNCTION_LOG_VOID(logLevelDebug);

    MEM_CONTEXT_TEMP_BEGIN()
    {
        TRY_BEGIN()
        {
            // PostgreSQL must be local
            pgIsLocalVerify();

            // Check the parameters
            if (strLstSize(cfgCommandParam()) < 1)
                THROW(ParamInvalidError, "at least one wal segment is required");

            LOG_INFO_FMT(
                "get %u WAL file(s) from archive: %s%s",
                strLstSize(cfgCommandParam()), strZ(strLstGet(cfgCommandParam(), 0)),
                strLstSize(cfgCommandParam()) == 1 ?
                    "" : zNewFmt("...%s", strZ(strLstGet(cfgCommandParam(), strLstSize(cfgCommandParam()) - 1))));

            // Check for archive files
            const ArchiveGetCheckResult checkResult = archiveGetCheck(cfgCommandParam());

            // If any files are missing get the first one (used to construct the "unable to find" warning)
            const String *archiveFileMissing = NULL;

            if (lstSize(checkResult.archiveFileMapList) < strLstSize(cfgCommandParam()))
                archiveFileMissing = strLstGet(cfgCommandParam(), lstSize(checkResult.archiveFileMapList));

            // Get archive files that were found
            if (!lstEmpty(checkResult.archiveFileMapList))
            {
                // Create the parallel executor
                ArchiveGetAsyncData jobData = {.archiveFileMapList = checkResult.archiveFileMapList};

                ProtocolParallel *const parallelExec = protocolParallelNew(
                    cfgOptionUInt64(cfgOptProtocolTimeout) / 2, archiveGetAsyncCallback, &jobData);

                for (unsigned int processIdx = 1; processIdx <= cfgOptionUInt(cfgOptProcessMax); processIdx++)
                    protocolParallelClientAdd(parallelExec, protocolLocalGet(protocolStorageTypeRepo, 0, processIdx));

                // Process jobs
                MEM_CONTEXT_TEMP_RESET_BEGIN()
                {
                    do
                    {
                        const unsigned int completed = protocolParallelProcess(parallelExec);

                        for (unsigned int jobIdx = 0; jobIdx < completed; jobIdx++)
                        {
                            // Get the job
                            ProtocolParallelJob *const job = protocolParallelResult(parallelExec);
                            const unsigned int processId = protocolParallelJobProcessId(job);

                            // Get wal segment name and archive file map
                            const String *const walSegment = varStr(protocolParallelJobKey(job));
                            const ArchiveFileMap *const fileMap = lstFind(checkResult.archiveFileMapList, &walSegment);
                            ASSERT(fileMap != NULL);

                            // Build warnings for status file
                            String *const warning = strNew();

                            if (!strLstEmpty(fileMap->warnList))
                                strCatFmt(warning, "%s", strZ(strLstJoin(fileMap->warnList, "\n")));

                            // The job was successful
                            if (protocolParallelJobErrorCode(job) == 0)
                            {
                                // Get the actual file retrieved
                                PackRead *const fileResult = protocolParallelJobResult(job);
                                const ArchiveGetFile *const file = lstGet(fileMap->actualList, pckReadU32P(fileResult));
                                ASSERT(file != NULL);

                                // Output file warnings
                                const StringList *const fileWarnList = pckReadStrLstP(fileResult);

                                for (unsigned int warnIdx = 0; warnIdx < strLstSize(fileWarnList); warnIdx++)
                                    LOG_WARN_PID(processId, strZ(strLstGet(fileWarnList, warnIdx)));

                                // Build file warnings for status file
                                if (!strLstEmpty(fileWarnList))
                                {
                                    strCatFmt(
                                        warning, "%s%s", strSize(warning) == 0 ? "" : "\n", strZ(strLstJoin(fileWarnList, "\n")));
                                }

                                if (strSize(warning) != 0)
                                    archiveAsyncStatusOkWrite(archiveModeGet, walSegment, warning);

                                LOG_DETAIL_PID_FMT(
                                    processId, FOUND_IN_REPO_ARCHIVE_MSG, strZ(walSegment),
                                    cfgOptionGroupName(cfgOptGrpRepo, file->repoIdx), strZ(file->archiveId));

                                // Rename temp WAL segment to actual name. This is done after the ok file is written so the ok file
                                // is guaranteed to exist before the foreground process finds the WAL segment.
                                storageMoveP(
                                    storageSpoolWrite(),
                                    storageNewReadP(
                                        storageSpool(),
                                        strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s." STORAGE_FILE_TEMP_EXT, strZ(walSegment))),
                                    storageNewWriteP(
                                        storageSpoolWrite(), strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s", strZ(walSegment))));
                            }
                            // Else the job errored
                            else
                            {
                                LOG_WARN_PID_FMT(
                                    processId, "[%s] %s", errorTypeName(errorTypeFromCode(protocolParallelJobErrorCode(job))),
                                    strZ(protocolParallelJobErrorMessage(job)));

                                archiveAsyncStatusErrorWrite(
                                    archiveModeGet, walSegment, protocolParallelJobErrorCode(job),
                                    strNewFmt(
                                        "%s%s", strZ(protocolParallelJobErrorMessage(job)),
                                        strSize(warning) == 0 ? "" : zNewFmt("\n%s", strZ(warning))));
                            }

                            protocolParallelJobFree(job);
                        }

                        // Reset the memory context occasionally so we don't use too much memory or slow down processing
                        MEM_CONTEXT_TEMP_RESET(1000);
                    }
                    while (!protocolParallelDone(parallelExec));
                }
                MEM_CONTEXT_TEMP_END();
            }

            // Log an error from archiveGetCheck() after any existing files have been fetched. This ordering is important because we
            // need to fetch as many valid files as possible before throwing an error.
            if (checkResult.errorType != NULL)
            {
                LOG_WARN_FMT("[%s] %s", errorTypeName(checkResult.errorType), strZ(checkResult.errorMessage));

                String *const message = strCat(strNew(), checkResult.errorMessage);

                if (!strLstEmpty(checkResult.warnList))
                    strCatFmt(message, "\n%s", strZ(strLstJoin(checkResult.warnList, "\n")));

                archiveAsyncStatusErrorWrite(
                    archiveModeGet, checkResult.errorFile, errorTypeCode(checkResult.errorType), message);
            }
            // If any files were missing write an ok file for the first missing file and add any warnings. It is important that this
            // happen right before the async process exits so the main process can immediately respawn the async process to retry
            // missing files.
            else if (archiveFileMissing != NULL)
            {
                LOG_DETAIL_FMT(UNABLE_TO_FIND_IN_ARCHIVE_MSG, strZ(archiveFileMissing));

                const String *message = NULL;

                if (!strLstEmpty(checkResult.warnList))
                    message = strLstJoin(checkResult.warnList, "\n");

                archiveAsyncStatusOkWrite(archiveModeGet, archiveFileMissing, message);
            }
        }
        // On any global error write a single error file to cover all unprocessed files
        CATCH_ANY()
        {
            archiveAsyncStatusErrorWrite(archiveModeGet, NULL, errorCode(), STR(errorMessage()));
            RETHROW();
        }
        TRY_END();
    }
    MEM_CONTEXT_TEMP_END();

    FUNCTION_LOG_RETURN_VOID();
}
