/***********************************************************************************************************************************
Test S3 Storage
***********************************************************************************************************************************/
#include <unistd.h>

#include "common/io/fdRead.h"
#include "common/io/fdWrite.h"
#include "storage/helper.h"
#include "version.h"

#include "common/harnessConfig.h"
#include "common/harnessFork.h"
#include "common/harnessServer.h"
#include "common/harnessStorage.h"

/***********************************************************************************************************************************
Constants
***********************************************************************************************************************************/
#define S3_TEST_HOST                                                "s3.amazonaws.com"

/***********************************************************************************************************************************
Calculate splitMax. This defines the point at which the maximum chunk size must be used in order to be able to upload the maximum
file size. The function is included here but can be used to calculate splitMax for other object stores as well by plugging in the
appropriate values.
***********************************************************************************************************************************/
#define TEST_STORAGE_S3_PART_SIZE_DEFAULT                           (5 * 1024 * 1024)
#define TEST_STORAGE_S3_PART_MAX                                    10000
#define TEST_STORAGE_S3_FILE_MAX                                    ((uint64_t)5 * 1024 * 1024 * 1024 * 1024)

static unsigned int
testStorageWriteChunkSplitMax(
    const size_t chunkSizeDefault, const size_t chunkSizeMax, const unsigned int chunkMax, const size_t chunkIncr,
    const unsigned int splitDefault, const uint64_t fileSizeMax)
{
    unsigned int splitIdx = chunkMax;

    for (; splitIdx >= splitDefault; splitIdx--)
    {
        uint64_t defaultTotal = (uint64_t)splitDefault * chunkSizeDefault;
        uint64_t maxTotal = (uint64_t)(chunkMax - splitIdx) * chunkSizeMax;

        // Calculate ascending chunks
        uint64_t ascTotal = 0;

        for (unsigned int chunkIdx = splitDefault; chunkIdx < splitIdx; chunkIdx++)
        {
            // Calculate ascending chunk size
            uint64_t chunkSize = (uint64_t)(chunkSizeMax - chunkIncr) * (chunkIdx - splitDefault + 1) / (splitIdx - splitDefault);

            // If ascending chunk size is less than default then return default
            if (chunkSize <= chunkSizeDefault)
            {
                chunkSize = chunkSizeDefault;
            }
            // Else if ascending chunk size is not evenly divisible by chunk increment then round up
            else if (chunkSize % chunkIncr != 0)
                chunkSize += chunkIncr - (chunkSize % chunkIncr);

            ascTotal += chunkSize;
        }

        if (defaultTotal + ascTotal + maxTotal > fileSizeMax)
            break;
    }

    return splitIdx;
}

/***********************************************************************************************************************************
Helper to build test requests
***********************************************************************************************************************************/
typedef struct TestRequestParam
{
    VAR_PARAM_HEADER;
    const char *content;
    const char *accessKey;
    const char *securityToken;
    const char *authorization;
    const char *range;
    const char *kms;
    const char *sseC;
    const char *ttl;
    const char *token;
    const char *tag;
    bool requesterPays;
} TestRequestParam;

#define testRequestP(write, s3, verb, path, ...)                                                                                   \
    testRequest(write, s3, verb, path, (TestRequestParam){VAR_PARAM_INIT, __VA_ARGS__})

static void
testRequest(IoWrite *write, Storage *s3, const char *verb, const char *path, TestRequestParam param)
{
    // Get security token from param
    const char *securityToken = param.securityToken == NULL ? NULL : param.securityToken;

    // If s3 storage is set then get the driver
    StorageS3 *driver = NULL;

    if (s3 != NULL)
    {
        driver = (StorageS3 *)storageDriver(s3);

        // Also update the security token if it is not already set
        if (param.securityToken == NULL)
            securityToken = strZNull(driver->securityToken);
    }

    // Add request
    String *request = strCatFmt(strNew(), "%s %s HTTP/1.1\r\nuser-agent:" PROJECT_NAME "/" PROJECT_VERSION "\r\n", verb, path);

    // Add authorization header when s3 service
    if (s3 != NULL)
    {
        strCatFmt(
            request,
            "authorization:AWS4-HMAC-SHA256 Credential=%s/\?\?\?\?\?\?\?\?/us-east-1/s3/aws4_request,SignedHeaders=",
            param.accessKey == NULL ? strZ(driver->accessKey) : param.accessKey);

        if (param.content != NULL)
            strCatZ(request, "content-md5;");

        strCatZ(request, "host;");

        if (param.range != NULL)
            strCatZ(request, "range;");

        strCatZ(request, "x-amz-content-sha256;x-amz-date");

        if (param.requesterPays)
            strCatZ(request, ";x-amz-request-payer");

        if (securityToken != NULL)
            strCatZ(request, ";x-amz-security-token");

        if (param.kms != NULL)
            strCatZ(request, ";x-amz-server-side-encryption;x-amz-server-side-encryption-aws-kms-key-id");

        if (param.sseC != NULL)
        {
            strCatZ(
                request,
                ";x-amz-server-side-encryption-customer-algorithm;x-amz-server-side-encryption-customer-key"
                ";x-amz-server-side-encryption-customer-key-md5");
        }

        if (param.tag != NULL)
            strCatZ(request, ";x-amz-tagging");

        strCatZ(request, ",Signature=????????????????????????????????????????????????????????????????\r\n");
    }

    // Add authorization
    if (param.authorization != NULL)
        strCatFmt(request, "authorization:%s\r\n", param.authorization);

    // Add content-length
    strCatFmt(request, "content-length:%zu\r\n", param.content != NULL ? strlen(param.content) : 0);

    // Add md5
    if (param.content != NULL)
    {
        strCatFmt(
            request, "content-md5:%s\r\n", strZ(strNewEncode(encodingBase64, cryptoHashOne(hashTypeMd5, BUFSTRZ(param.content)))));
    }

    // Add host
    if (s3 != NULL)
    {
        if (driver->uriStyle == storageS3UriStyleHost)
            strCatZ(request, "host:bucket." S3_TEST_HOST "\r\n");
        else
            strCatZ(request, "host:" S3_TEST_HOST "\r\n");
    }
    else
        strCatFmt(request, "host:%s\r\n", strZ(hrnServerHost()));

    // Add range
    if (param.range != NULL)
        strCatFmt(request, "range:bytes=%s\r\n", param.range);

    // Add content checksum and date if s3 service
    if (s3 != NULL)
    {
        // Add content sha256 and date
        strCatFmt(
            request,
            "x-amz-content-sha256:%s\r\n"
            "x-amz-date:????????T??????Z" "\r\n",
            param.content == NULL ?
                HASH_TYPE_SHA256_ZERO : strZ(strNewEncode(encodingHex, cryptoHashOne(hashTypeSha256, BUFSTRZ(param.content)))));

        // Add security token
        if (securityToken != NULL)
            strCatFmt(request, "x-amz-security-token:%s\r\n", securityToken);
    }

    // Requestor pays
    if (param.requesterPays)
        strCatZ(request, "x-amz-request-payer:requester\r\n");

    // Add kms key
    if (param.kms != NULL)
    {
        strCatZ(request, "x-amz-server-side-encryption:aws:kms\r\n");
        strCatFmt(request, "x-amz-server-side-encryption-aws-kms-key-id:%s\r\n", param.kms);
    }

    // Add sseC key
    if (param.sseC != NULL)
    {
        strCatZ(request, "x-amz-server-side-encryption-customer-algorithm:AES256\r\n");
        strCatFmt(request, "x-amz-server-side-encryption-customer-key:%s\r\n", param.sseC);
        strCatFmt(
            request, "x-amz-server-side-encryption-customer-key-md5:%s\r\n",
            strZ(strNewEncode(encodingBase64, cryptoHashOne(hashTypeMd5, bufNewDecode(encodingBase64, STR(param.sseC))))));
    }

    // Add tags
    if (param.tag != NULL)
        strCatFmt(request, "x-amz-tagging:%s\r\n", param.tag);

    // Add metadata token
    if (param.token != NULL)
        strCatFmt(request, "x-aws-ec2-metadata-token:%s\r\n", param.token);

    // Add metadata token ttl
    if (param.ttl != NULL)
        strCatFmt(request, "x-aws-ec2-metadata-token-ttl-seconds:%s\r\n", param.ttl);

    // Add final \r\n
    strCatZ(request, "\r\n");

    // Add content
    if (param.content != NULL)
        strCatZ(request, param.content);

    hrnServerScriptExpect(write, request);
}

/***********************************************************************************************************************************
Helper to build test responses
***********************************************************************************************************************************/
typedef struct TestResponseParam
{
    VAR_PARAM_HEADER;
    unsigned int code;
    const char *http;
    const char *header;
    const char *content;
    const Variant *contentSize;
} TestResponseParam;

#define testResponseP(write, ...)                                                                                                  \
    testResponse(write, (TestResponseParam){VAR_PARAM_INIT, __VA_ARGS__})

static void
testResponse(IoWrite *write, TestResponseParam param)
{
    // Set code to 200 if not specified
    param.code = param.code == 0 ? 200 : param.code;

    // Output header and code
    String *response = strCatFmt(strNew(), "HTTP/%s %u ", param.http == NULL ? "1.1" : param.http, param.code);

    // Add reason for some codes
    switch (param.code)
    {
        case 200:
            strCatZ(response, "OK");
            break;
    }

    // End header
    strCatZ(response, "\r\n");

    // Headers
    if (param.header != NULL)
        strCatFmt(response, "%s\r\n", param.header);

    // Content
    if (param.content != NULL)
    {
        strCatFmt(
            response,
            "content-length:%zu\r\n"
            "\r\n"
            "%s",
            param.contentSize != NULL ? varUInt(param.contentSize) : strlen(param.content), param.content);
    }
    else
        strCatZ(response, "\r\n");

    hrnServerScriptReply(write, response);
}

/***********************************************************************************************************************************
Format ISO-8601 date with - and :
***********************************************************************************************************************************/
static String *
testS3DateTime(time_t time)
{
    FUNCTION_HARNESS_BEGIN();
        FUNCTION_HARNESS_PARAM(TIME, time);
    FUNCTION_HARNESS_END();

    FUNCTION_HARNESS_RETURN(STRING, strNewTimeP("%Y-%m-%dT%H:%M:%SZ", time, .utc = true));
}

/***********************************************************************************************************************************
Test Run
***********************************************************************************************************************************/
static void
testRun(void)
{
    FUNCTION_HARNESS_VOID();

    // Set storage helper
    static const StorageHelper storageHelperList[] = {STORAGE_S3_HELPER, STORAGE_END_HELPER};
    storageHelperInit(storageHelperList);

    // Test strings
    const String *path = STRDEF("/");
    const String *bucket = STRDEF("bucket");
    const String *region = STRDEF("us-east-1");
    const String *endPoint = STRDEF("s3.amazonaws.com");
    const String *host = hrnServerHost();
    const String *accessKey = STRDEF("AKIAIOSFODNN7EXAMPLE");
    const String *secretAccessKey = STRDEF("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY");
    const String *securityToken = STRDEF(
        "AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/q"
        "kPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xV"
        "qr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA==");
    const String *credRole = STRDEF("credrole");

    // Config settings that are required for every test (without endpoint for special tests)
    StringList *commonArgWithoutEndpointList = strLstNew();
    hrnCfgArgRawZ(commonArgWithoutEndpointList, cfgOptStanza, "db");
    hrnCfgArgRawZ(commonArgWithoutEndpointList, cfgOptRepoType, "s3");
    hrnCfgArgRaw(commonArgWithoutEndpointList, cfgOptRepoPath, path);
    hrnCfgArgRaw(commonArgWithoutEndpointList, cfgOptRepoS3Bucket, bucket);
    hrnCfgArgRaw(commonArgWithoutEndpointList, cfgOptRepoS3Region, region);

    // TLS can only be verified in a container
    if (!TEST_IN_CONTAINER)
        hrnCfgArgRawBool(commonArgWithoutEndpointList, cfgOptRepoStorageVerifyTls, false);

    // Config settings that are required for every test (with endpoint)
    StringList *commonArgList = strLstDup(commonArgWithoutEndpointList);
    hrnCfgArgRaw(commonArgList, cfgOptRepoS3Endpoint, endPoint);

    // Secure options must be loaded into environment variables
    hrnCfgEnvRaw(cfgOptRepoS3Key, accessKey);
    hrnCfgEnvRaw(cfgOptRepoS3KeySecret, secretAccessKey);

    // *****************************************************************************************************************************
    if (testBegin("storageS3Auth()"))
    {
        // -------------------------------------------------------------------------------------------------------------------------
        TEST_TITLE("config without token");

        char logBuf[STACK_TRACE_PARAM_MAX];
        StringList *argList = strLstDup(commonArgList);
        HRN_CFG_LOAD(cfgCmdArchivePush, argList);

        StorageS3 *driver = (StorageS3 *)storageDriver(storageRepoGet(0, false));

        TEST_RESULT_STR(driver->bucket, bucket, "check bucket");
        TEST_RESULT_STR(driver->region, region, "check region");
        TEST_RESULT_STR(driver->bucketEndpoint, strNewFmt("%s.%s", strZ(bucket), strZ(endPoint)), "check host");
        TEST_RESULT_STR(driver->accessKey, accessKey, "check access key");
        TEST_RESULT_STR(driver->secretAccessKey, secretAccessKey, "check secret access key");
        TEST_RESULT_STR(driver->securityToken, NULL, "check security token");

        TEST_RESULT_VOID(FUNCTION_LOG_OBJECT_FORMAT(driver->httpClient, httpClientToLog, logBuf, sizeof(logBuf)), "httpClientToLog");
        TEST_RESULT_Z(
            logBuf,
            zNewFmt(
                "{ioClient: {type: tls, driver: {ioClient: {type: socket, driver: {host: bucket.s3.amazonaws.com, port: 443"
                ", timeoutConnect: 60000, timeoutSession: 60000}}, timeoutConnect: 60000, timeoutSession: 60000"
                ", verifyPeer: %s}}, reusable: 0, timeout: 60000}",
                cvtBoolToConstZ(TEST_IN_CONTAINER)),
            "check http client");

        // -------------------------------------------------------------------------------------------------------------------------
        TEST_TITLE("auth with token");

        HttpHeader *header = httpHeaderNew(NULL);
        HttpQuery *query = httpQueryNewP();
        httpQueryAdd(query, STRDEF("list-type"), STRDEF("2"));

        TEST_RESULT_VOID(
            storageS3Auth(
                driver, STRDEF("GET"), STRDEF("/"), query, STRDEF("20170606T121212Z"), header, STRDEF(HASH_TYPE_SHA256_ZERO)),
            "generate authorization");
        TEST_RESULT_STR_Z(
            httpHeaderGet(header, STRDEF("authorization")),
            "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20170606/us-east-1/s3/aws4_request"
            ",SignedHeaders=host;x-amz-content-sha256;x-amz-date"
            ",Signature=cb03bf1d575c1f8904dabf0e573990375340ab293ef7ad18d049fc1338fd89b3",
            "check authorization header");

        // Test again to be sure cache signing key is used
        const Buffer *lastSigningKey = driver->signingKey;

        TEST_RESULT_VOID(
            storageS3Auth(
                driver, STRDEF("GET"), STRDEF("/"), query, STRDEF("20170606T121212Z"), header, STRDEF(HASH_TYPE_SHA256_ZERO)),
            "generate authorization");
        TEST_RESULT_STR_Z(
            httpHeaderGet(header, STRDEF("authorization")),
            "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20170606/us-east-1/s3/aws4_request"
            ",SignedHeaders=host;x-amz-content-sha256;x-amz-date"
            ",Signature=cb03bf1d575c1f8904dabf0e573990375340ab293ef7ad18d049fc1338fd89b3",
            "check authorization header");
        TEST_RESULT_BOOL(driver->signingKey == lastSigningKey, true, "check signing key was reused");

        // -------------------------------------------------------------------------------------------------------------------------
        TEST_TITLE("change date to generate new signing key");

        TEST_RESULT_VOID(
            storageS3Auth(
                driver, STRDEF("GET"), STRDEF("/"), query, STRDEF("20180814T080808Z"), header, STRDEF(HASH_TYPE_SHA256_ZERO)),
            "generate authorization");
        TEST_RESULT_STR_Z(
            httpHeaderGet(header, STRDEF("authorization")),
            "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20180814/us-east-1/s3/aws4_request"
            ",SignedHeaders=host;x-amz-content-sha256;x-amz-date"
            ",Signature=d0fa9c36426eb94cdbaf287a7872c7a3b6c913f523163d0d7debba0758e36f49",
            "check authorization header");
        TEST_RESULT_BOOL(driver->signingKey != lastSigningKey, true, "check signing key was regenerated");

        // -------------------------------------------------------------------------------------------------------------------------
        TEST_TITLE("config with token, endpoint with custom port, and ca-file/path");

        argList = strLstDup(commonArgWithoutEndpointList);
        hrnCfgArgRawZ(argList, cfgOptRepoS3Endpoint, "custom.endpoint:333");
        hrnCfgArgRawZ(argList, cfgOptRepoStorageCaPath, "/path/to/cert");
        hrnCfgArgRawZ(argList, cfgOptRepoStorageCaFile, HRN_SERVER_CA);
        hrnCfgEnvRaw(cfgOptRepoS3Token, securityToken);
        HRN_CFG_LOAD(cfgCmdArchivePush, argList);

        driver = (StorageS3 *)storageDriver(storageRepoGet(0, false));

        TEST_RESULT_STR(driver->securityToken, securityToken, "check security token");
        TEST_RESULT_VOID(
            FUNCTION_LOG_OBJECT_FORMAT(driver->httpClient, httpClientToLog, logBuf, sizeof(logBuf)), "httpClientToLog");
        TEST_RESULT_Z(
            logBuf,
            zNewFmt(
                "{ioClient: {type: tls, driver: {ioClient: {type: socket, driver: {host: bucket.custom.endpoint, port: 333"
                ", timeoutConnect: 60000, timeoutSession: 60000}}, timeoutConnect: 60000, timeoutSession: 60000"
                ", verifyPeer: %s}}, reusable: 0, timeout: 60000}",
                cvtBoolToConstZ(TEST_IN_CONTAINER)),
            "check http client");

        // -------------------------------------------------------------------------------------------------------------------------
        TEST_TITLE("auth with token");

        TEST_RESULT_VOID(
            storageS3Auth(
                driver, STRDEF("GET"), STRDEF("/"), query, STRDEF("20170606T121212Z"), header, STRDEF(HASH_TYPE_SHA256_ZERO)),
            "generate authorization");
        TEST_RESULT_STR_Z(
            httpHeaderGet(header, STRDEF("authorization")),
            "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20170606/us-east-1/s3/aws4_request"
            ",SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token"
            ",Signature=85278841678ccbc0f137759265030d7b5e237868dd36eea658426b18344d1685",
            "check authorization header");
    }

    // *****************************************************************************************************************************
    if (testBegin("StorageS3, StorageReadS3, and StorageWriteS3 with HTTPS"))
    {
        HRN_FORK_BEGIN()
        {
            const unsigned int testPort = hrnServerPortNext();
            const unsigned int testPortAuth = hrnServerPortNext();

            HRN_FORK_CHILD_BEGIN(.prefix = "s3 server", .timeout = 5000)
            {
                TEST_RESULT_VOID(hrnServerRunP(HRN_FORK_CHILD_READ(), hrnServerProtocolTls, testPort), "s3 server");
            }
            HRN_FORK_CHILD_END();

            HRN_FORK_CHILD_BEGIN(.prefix = "auth server", .timeout = 15000)
            {
                TEST_RESULT_VOID(hrnServerRunP(HRN_FORK_CHILD_READ(), hrnServerProtocolSocket, testPortAuth), "auth server");
            }
            HRN_FORK_CHILD_END();

            HRN_FORK_PARENT_BEGIN()
            {
                // Do not use HRN_FORK_PARENT_WRITE() here so individual names can be assigned to help with debugging
                IoWrite *service = hrnServerScriptBegin(
                    ioFdWriteNewOpen(STRDEF("s3 client write"), HRN_FORK_PARENT_WRITE_FD(0), 2000));
                IoWrite *auth = hrnServerScriptBegin(
                    ioFdWriteNewOpen(STRDEF("auth client write"), HRN_FORK_PARENT_WRITE_FD(1), 2000));

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("config with keys, token, and host with custom port");

                StringList *argList = strLstDup(commonArgList);
                hrnCfgArgRawFmt(argList, cfgOptRepoStorageHost, "https://%s:%u", strZ(host), testPort);
                hrnCfgEnvRaw(cfgOptRepoS3Token, securityToken);
                HRN_CFG_LOAD(cfgCmdArchivePush, argList);

                Storage *s3 = storageRepoGet(0, true);
                StorageS3 *driver = (StorageS3 *)storageDriver(s3);

                TEST_RESULT_STR(s3->path, path, "check path");
                TEST_RESULT_BOOL(storageFeature(s3, storageFeaturePath), false, "check path feature");
                TEST_RESULT_UINT(driver->partSize, 5 * 1024 * 1024, "check part size");

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("check part sizes");

                unsigned int splitDefault = (unsigned int)((1024UL * 1024 * 1024) / TEST_STORAGE_S3_PART_SIZE_DEFAULT) + 1;
                TEST_RESULT_UINT(splitDefault, STORAGE_S3_SPLIT_DEFAULT, "check part default split");

                unsigned int splitMax = testStorageWriteChunkSplitMax(
                    TEST_STORAGE_S3_PART_SIZE_DEFAULT, STORAGE_CHUNK_SIZE_MAX, TEST_STORAGE_S3_PART_MAX, STORAGE_CHUNK_INCR,
                    splitDefault, TEST_STORAGE_S3_FILE_MAX);
                TEST_RESULT_UINT(splitMax, STORAGE_S3_SPLIT_MAX, "check part max split");

                uint64_t sizeCheck = 0;

                for (unsigned int chunkIdx = 0; chunkIdx < TEST_STORAGE_S3_PART_MAX; chunkIdx++)
                {
                    sizeCheck += storageWriteChunkSize(
                        TEST_STORAGE_S3_PART_SIZE_DEFAULT, STORAGE_S3_SPLIT_DEFAULT, STORAGE_S3_SPLIT_MAX, chunkIdx);
                }

                TEST_RESULT_BOOL(sizeCheck >= TEST_STORAGE_S3_FILE_MAX, true, "check part total min");
                TEST_RESULT_BOOL(sizeCheck < TEST_STORAGE_S3_FILE_MAX + STORAGE_CHUNK_SIZE_MAX, true, "check part total max");

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("coverage for noop functions");

                TEST_RESULT_VOID(storagePathSyncP(s3, STRDEF("path")), "path sync is a noop");

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("ignore missing file");

                hrnServerScriptAccept(service);
                testRequestP(service, s3, HTTP_VERB_GET, "/fi%26le.txt");
                testResponseP(service, .code = 404);

                TEST_RESULT_PTR(storageGetP(storageNewReadP(s3, STRDEF("fi&le.txt"), .ignoreMissing = true)), NULL, "get file");

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("get file with retry");

                testRequestP(service, s3, HTTP_VERB_GET, "/file.txt");
                testResponseP(service, .content = "12345678911234567892", .contentSize = VARUINT(30));

                hrnServerScriptClose(service);
                hrnServerScriptAccept(service);

                testRequestP(service, s3, HTTP_VERB_GET, "/file.txt", .range = "20-");
                testResponseP(service, .content = "1234567893");

                const size_t ioBufferSizeDefault = ioBufferSize();
                ioBufferSizeSet(20);

                TEST_RESULT_STR_Z(
                    strNewBuf(storageGetP(storageNewReadP(s3, STRDEF("file.txt")))),
                    "123456789112345678921234567893", "get file");

                ioBufferSizeSet(ioBufferSizeDefault);

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("get file with retry, offset, and limit");

                testRequestP(service, s3, HTTP_VERB_GET, "/file.txt", .range = "1-29");
                testResponseP(service, .content = "23456789112345678921X", .contentSize = VARUINT(30));

                hrnServerScriptAbort(service);
                hrnServerScriptAccept(service);

                testRequestP(service, s3, HTTP_VERB_GET, "/file.txt", .range = "21-29");
                testResponseP(service, .content = "23456789");

                ioBufferSizeSet(20);

                TEST_RESULT_STR_Z(
                    strNewBuf(storageGetP(storageNewReadP(s3, STRDEF("file.txt"), .offset = 1, .limit = VARUINT64(29)))),
                    "2345678911234567892123456789", "get file");

                ioBufferSizeSet(ioBufferSizeDefault);

                // Close to reset buffer size
                hrnServerScriptClose(service);
                hrnServerScriptAccept(service);

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("get file all retries fail");

                testRequestP(service, s3, HTTP_VERB_GET, "/file.txt", .range = "1-29");
                testResponseP(service, .content = "X", .contentSize = VARUINT(30));

                hrnServerScriptClose(service);
                hrnServerScriptAccept(service);

                testRequestP(service, s3, HTTP_VERB_GET, "/file.txt", .range = "1-29");
                testResponseP(service, .content = "X", .contentSize = VARUINT(30));

                hrnServerScriptClose(service);
                hrnServerScriptAccept(service);

                testRequestP(service, s3, HTTP_VERB_GET, "/file.txt", .range = "1-29");
                testResponseP(service, .content = "X", .contentSize = VARUINT(30));

                hrnServerScriptClose(service);
                hrnServerScriptAccept(service);

                bool errorCaught = false;

                TRY_BEGIN()
                {
                    storageGetP(storageNewReadP(s3, STRDEF("file.txt"), .offset = 1, .limit = VARUINT64(29)));
                }
                CATCH_ANY()
                {
                    errorCaught = true;
                }
                TRY_END();

                TEST_RESULT_BOOL(errorCaught, true, "check error was caught");

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("get zero-length file");

                testRequestP(service, s3, HTTP_VERB_GET, "/file0.txt");
                testResponseP(service);

                TEST_RESULT_STR_Z(strNewBuf(storageGetP(storageNewReadP(s3, STRDEF("file0.txt")))), "", "get zero-length file");

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("switch to temp credentials");

                hrnServerScriptClose(service);

                argList = strLstDup(commonArgList);
                hrnCfgArgRawFmt(argList, cfgOptRepoStorageHost, "%s:%u", strZ(host), testPort);
                hrnCfgArgRaw(argList, cfgOptRepoS3Role, credRole);
                hrnCfgArgRawStrId(argList, cfgOptRepoS3KeyType, storageS3KeyTypeAuto);
                hrnCfgArgRawZ(argList, cfgOptRepoS3KmsKeyId, "kmskey1");
                hrnCfgEnvRawZ(cfgOptRepoS3SseCustomerKey, "rA1P");
                hrnCfgArgRawZ(argList, cfgOptRepoStorageTag, "Key1=Value1");
                hrnCfgArgRawZ(argList, cfgOptRepoStorageTag, " Key 2= Value 2");
                HRN_CFG_LOAD(cfgCmdArchivePush, argList);
                hrnCfgEnvRemoveRaw(cfgOptRepoS3SseCustomerKey);

                s3 = storageRepoGet(0, true);
                driver = (StorageS3 *)storageDriver(s3);

                TEST_RESULT_STR(s3->path, path, "check path");
                TEST_RESULT_STR(driver->credRole, credRole, "check role");
                TEST_RESULT_BOOL(storageFeature(s3, storageFeaturePath), false, "check path feature");

                // Set partSize to a small value for testing
                driver->partSize = 16;

                // Testing requires the auth http client to be redirected
                driver->credHost = hrnServerHost();
                driver->credHttpClient = httpClientNew(sckClientNew(host, testPortAuth, 5000, 5000), 5000);

                // Now that we have checked the role when set explicitly, null it out to make sure it is retrieved automatically
                driver->credRole = NULL;

                hrnServerScriptAccept(service);

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("error when retrieving role");

                hrnServerScriptAccept(auth);

                testRequestP(auth, NULL, HTTP_VERB_PUT, "/latest/api/token", .ttl = "15");
                testResponseP(auth, .http = "1.0", .content = "WtokenW");

                hrnServerScriptClose(auth);
                hrnServerScriptAccept(auth);

                testRequestP(auth, NULL, HTTP_VERB_GET, S3_CREDENTIAL_PATH, .token = "WtokenW");
                testResponseP(auth, .http = "1.0", .code = 301);

                hrnServerScriptClose(auth);

                TEST_ERROR_FMT(
                    storageGetP(storageNewReadP(s3, STRDEF("file.txt"))), ProtocolError,
                    "HTTP request failed with 301:\n"
                    "*** Path/Query ***:\n"
                    "GET /latest/meta-data/iam/security-credentials\n"
                    "*** Request Headers ***:\n"
                    "content-length: 0\n"
                    "host: %s\n"
                    "x-aws-ec2-metadata-token: WtokenW",
                    strZ(hrnServerHost()));

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("missing role and token");

                hrnServerScriptAccept(auth);

                testRequestP(auth, NULL, HTTP_VERB_PUT, "/latest/api/token", .ttl = "15");
                testResponseP(auth, .http = "1.0", .code = 301);

                hrnServerScriptClose(auth);
                hrnServerScriptAccept(auth);

                testRequestP(auth, NULL, HTTP_VERB_GET, S3_CREDENTIAL_PATH);
                testResponseP(auth, .http = "1.0", .code = 404);

                hrnServerScriptClose(auth);

                TEST_ERROR(
                    storageGetP(storageNewReadP(s3, STRDEF("file.txt"))), ProtocolError,
                    "role to retrieve temporary credentials not found\n"
                    "HINT: is a valid IAM role associated with this instance?");

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("error when retrieving temp credentials and missing token");

                hrnServerScriptAccept(auth);

                testRequestP(auth, NULL, HTTP_VERB_PUT, "/latest/api/token", .ttl = "15");
                testResponseP(auth, .http = "1.0", .code = 301);

                hrnServerScriptClose(auth);
                hrnServerScriptAccept(auth);

                testRequestP(auth, NULL, HTTP_VERB_GET, S3_CREDENTIAL_PATH);
                testResponseP(auth, .http = "1.0", .content = strZ(credRole));

                hrnServerScriptClose(auth);
                hrnServerScriptAccept(auth);

                testRequestP(auth, NULL, HTTP_VERB_GET, zNewFmt(S3_CREDENTIAL_PATH "/%s", strZ(credRole)));
                testResponseP(auth, .http = "1.0", .code = 300);

                hrnServerScriptClose(auth);

                TEST_ERROR_FMT(
                    storageGetP(storageNewReadP(s3, STRDEF("file.txt"))), ProtocolError,
                    "HTTP request failed with 300:\n"
                    "*** Path/Query ***:\n"
                    "GET /latest/meta-data/iam/security-credentials/credrole\n"
                    "*** Request Headers ***:\n"
                    "content-length: 0\n"
                    "host: %s",
                    strZ(hrnServerHost()));

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("invalid temp credentials role and missing token");

                hrnServerScriptAccept(auth);

                testRequestP(auth, NULL, HTTP_VERB_PUT, "/latest/api/token", .ttl = "15");
                testResponseP(auth, .http = "1.0", .code = 301);

                hrnServerScriptClose(auth);
                hrnServerScriptAccept(auth);

                testRequestP(auth, NULL, HTTP_VERB_GET, zNewFmt(S3_CREDENTIAL_PATH "/%s", strZ(credRole)));
                testResponseP(auth, .http = "1.0", .code = 404);

                hrnServerScriptClose(auth);

                TEST_ERROR_FMT(
                    storageGetP(storageNewReadP(s3, STRDEF("file.txt"))), ProtocolError,
                    "role '%s' not found\n"
                    "HINT: is '%s' a valid IAM role associated with this instance?",
                    strZ(credRole), strZ(credRole));

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("invalid code when retrieving temp credentials");

                hrnServerScriptAccept(auth);

                testRequestP(auth, NULL, HTTP_VERB_PUT, "/latest/api/token", .ttl = "15");
                testResponseP(auth, .http = "1.0", .content = "XtokenX");

                hrnServerScriptClose(auth);
                hrnServerScriptAccept(auth);

                testRequestP(
                    auth, NULL, HTTP_VERB_GET, zNewFmt(S3_CREDENTIAL_PATH "/%s", strZ(credRole)), .token = "XtokenX");
                testResponseP(auth, .http = "1.0", .content = "{\"Code\":\"IAM role is not configured\"}");

                hrnServerScriptClose(auth);

                TEST_ERROR(
                    storageGetP(storageNewReadP(s3, STRDEF("file.txt"))), FormatError,
                    "unable to retrieve temporary credentials: IAM role is not configured");

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("non-404 error");

                hrnServerScriptAccept(auth);

                testRequestP(auth, NULL, HTTP_VERB_PUT, "/latest/api/token", .ttl = "15");
                testResponseP(auth, .http = "1.0", .content = "YtokenY");

                hrnServerScriptClose(auth);
                hrnServerScriptAccept(auth);

                testRequestP(
                    auth, NULL, HTTP_VERB_GET, zNewFmt(S3_CREDENTIAL_PATH "/%s", strZ(credRole)), .token = "YtokenY");
                testResponseP(
                    auth,
                    .content = zNewFmt(
                        "{\"Code\":\"Success\",\"AccessKeyId\":\"x\",\"SecretAccessKey\":\"y\",\"Token\":\"z\""
                        ",\"Expiration\":\"%s\"}",
                        strZ(testS3DateTime(time(NULL) + (S3_CREDENTIAL_RENEW_SEC - 1)))));

                hrnServerScriptClose(auth);

                testRequestP(service, s3, HTTP_VERB_GET, "/file.txt", .accessKey = "x", .securityToken = "z", .sseC = "rA1P");
                testResponseP(service, .code = 303, .content = "CONTENT");

                StorageRead *read = NULL;
                TEST_ASSIGN(read, storageNewReadP(s3, STRDEF("file.txt"), .ignoreMissing = true), "new read file");
                TEST_RESULT_BOOL(storageReadIgnoreMissing(read), true, "check ignore missing");
                TEST_RESULT_STR_Z(storageReadName(read), "/file.txt", "check name");

                TEST_ERROR(
                    ioReadOpen(storageReadIo(read)), ProtocolError,
                    "HTTP request failed with 303:\n"
                    "*** Path/Query ***:\n"
                    "GET /file.txt\n"
                    "*** Request Headers ***:\n"
                    "authorization: <redacted>\n"
                    "content-length: 0\n"
                    "host: bucket." S3_TEST_HOST "\n"
                    "x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n"
                    "x-amz-date: <redacted>\n"
                    "x-amz-security-token: <redacted>\n"
                    "x-amz-server-side-encryption-customer-algorithm: AES256\n"
                    "x-amz-server-side-encryption-customer-key: <redacted>\n"
                    "x-amz-server-side-encryption-customer-key-md5: <redacted>\n"
                    "*** Response Headers ***:\n"
                    "content-length: 7\n"
                    "*** Response Content ***:\n"
                    "CONTENT")

                // Check that temp credentials were set
                TEST_RESULT_STR_Z(driver->accessKey, "x", "check access key");
                TEST_RESULT_STR_Z(driver->secretAccessKey, "y", "check secret access key");
                TEST_RESULT_STR_Z(driver->securityToken, "z", "check security token");

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("write file in one part");

                hrnServerScriptAccept(auth);

                testRequestP(auth, NULL, HTTP_VERB_PUT, "/latest/api/token", .ttl = "15");
                testResponseP(auth, .http = "1.0", .content = "ZtokenZ");

                hrnServerScriptClose(auth);
                hrnServerScriptAccept(auth);

                testRequestP(auth, NULL, HTTP_VERB_GET, zNewFmt(S3_CREDENTIAL_PATH "/%s", strZ(credRole)), .token = "ZtokenZ");
                testResponseP(
                    auth,
                    .content = zNewFmt(
                        "{\"Code\":\"Success\",\"AccessKeyId\":\"xx\",\"SecretAccessKey\":\"yy\",\"Token\":\"zz\""
                        ",\"Expiration\":\"%s\"}",
                        strZ(testS3DateTime(time(NULL) + (S3_CREDENTIAL_RENEW_SEC * 2)))));

                hrnServerScriptClose(auth);

                testRequestP(
                    service, s3, HTTP_VERB_PUT, "/file.txt", .content = "ABCD", .accessKey = "xx", .securityToken = "zz",
                    .kms = "kmskey1", .sseC = "rA1P", .tag = "%20Key%202=%20Value%202&Key1=Value1");
                testResponseP(service);

                // Make a copy of the signing key to verify that it gets changed when the keys are updated
                const Buffer *oldSigningKey = bufDup(driver->signingKey);

                StorageWrite *write = NULL;
                TEST_ASSIGN(write, storageNewWriteP(s3, STRDEF("file.txt")), "new write");
                TEST_RESULT_VOID(storagePutP(write, BUFSTRDEF("ABCD")), "write");

                TEST_RESULT_BOOL(storageWriteAtomic(write), true, "write is atomic");
                TEST_RESULT_BOOL(storageWriteCreatePath(write), true, "path will be created");
                TEST_RESULT_UINT(storageWriteModeFile(write), 0, "file mode is 0");
                TEST_RESULT_UINT(storageWriteModePath(write), 0, "path mode is 0");
                TEST_RESULT_STR_Z(storageWriteName(write), "/file.txt", "check file name");
                TEST_RESULT_BOOL(storageWriteSyncFile(write), true, "file is synced");
                TEST_RESULT_BOOL(storageWriteSyncPath(write), true, "path is synced");
                TEST_RESULT_BOOL(storageWriteTruncate(write), true, "file will be truncated");

                TEST_RESULT_VOID(storageWriteS3Close(ioWriteDriver(storageWriteIo(write))), "close file again");

                // Check that temp credentials were changed
                TEST_RESULT_STR_Z(driver->accessKey, "xx", "check access key");
                TEST_RESULT_STR_Z(driver->secretAccessKey, "yy", "check secret access key");
                TEST_RESULT_STR_Z(driver->securityToken, "zz", "check security token");

                // Check that the signing key changed
                TEST_RESULT_BOOL(bufEq(driver->signingKey, oldSigningKey), false, "signing key changed");

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("write zero-length file");

                testRequestP(
                    service, s3, HTTP_VERB_PUT, "/file.txt", .content = "", .kms = "kmskey1", .sseC = "rA1P",
                    .tag = "%20Key%202=%20Value%202&Key1=Value1");
                testResponseP(service);

                TEST_ASSIGN(write, storageNewWriteP(s3, STRDEF("file.txt")), "new write");
                TEST_RESULT_VOID(storagePutP(write, NULL), "write");

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("write file in chunks with nothing left over on close");

                testRequestP(
                    service, s3, HTTP_VERB_POST, "/file.txt?uploads=", .kms = "kmskey1", .sseC = "rA1P",
                    .tag = "%20Key%202=%20Value%202&Key1=Value1");
                testResponseP(
                    service,
                    .content =
                        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
                        "<InitiateMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
                        "<Bucket>bucket</Bucket>"
                        "<Key>file.txt</Key>"
                        "<UploadId>WxRt</UploadId>"
                        "</InitiateMultipartUploadResult>");

                testRequestP(
                    service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=1&uploadId=WxRt", .content = "1234567890123456",
                    .sseC = "rA1P");
                testResponseP(service, .header = "etag:WxRt1");

                testRequestP(
                    service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=2&uploadId=WxRt", .content = "7890123456789012",
                    .sseC = "rA1P");
                testResponseP(service, .header = "eTag:WxRt2");

                testRequestP(
                    service, s3, HTTP_VERB_POST, "/file.txt?uploadId=WxRt",
                    .content =
                        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
                        "<CompleteMultipartUpload>"
                        "<Part><PartNumber>1</PartNumber><ETag>WxRt1</ETag></Part>"
                        "<Part><PartNumber>2</PartNumber><ETag>WxRt2</ETag></Part>"
                        "</CompleteMultipartUpload>\n");
                testResponseP(
                    service,
                    .content =
                        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
                        "<CompleteMultipartUploadResult><ETag>XXX</ETag></CompleteMultipartUploadResult>");

                TEST_ASSIGN(write, storageNewWriteP(s3, STRDEF("file.txt")), "new write");
                TEST_RESULT_VOID(storagePutP(write, BUFSTRDEF("12345678901234567890123456789012")), "write");

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("error in success response of multipart upload");

                // Stop writing tags
                driver->tag = NULL;

                testRequestP(service, s3, HTTP_VERB_POST, "/file.txt?uploads=", .kms = "kmskey1", .sseC = "rA1P");
                testResponseP(
                    service,
                    .content =
                        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
                        "<InitiateMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
                        "<Bucket>bucket</Bucket>"
                        "<Key>file.txt</Key>"
                        "<UploadId>WxRt</UploadId>"
                        "</InitiateMultipartUploadResult>");

                testRequestP(
                    service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=1&uploadId=WxRt", .content = "1234567890123456",
                    .sseC = "rA1P");
                testResponseP(service, .header = "etag:WxRt1");

                testRequestP(
                    service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=2&uploadId=WxRt", .content = "7890123456789012",
                    .sseC = "rA1P");
                testResponseP(service, .header = "eTag:WxRt2");

                testRequestP(
                    service, s3, HTTP_VERB_POST, "/file.txt?uploadId=WxRt",
                    .content =
                        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
                        "<CompleteMultipartUpload>"
                        "<Part><PartNumber>1</PartNumber><ETag>WxRt1</ETag></Part>"
                        "<Part><PartNumber>2</PartNumber><ETag>WxRt2</ETag></Part>"
                        "</CompleteMultipartUpload>\n");
                testResponseP(
                    service,
                    .content =
                        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
                        "<Error><Code>AccessDenied</Code><Message>Access Denied</Message></Error>");

                TEST_ASSIGN(write, storageNewWriteP(s3, STRDEF("file.txt")), "new write");
                TEST_ERROR(
                    storagePutP(write, BUFSTRDEF("12345678901234567890123456789012")), ProtocolError,
                    "HTTP request failed with 200 (OK):\n"
                    "*** Path/Query ***:\n"
                    "POST /file.txt?uploadId=WxRt\n"
                    "*** Request Headers ***:\n"
                    "authorization: <redacted>\n"
                    "content-length: 205\n"
                    "content-md5: 37smUM6Ah2/EjZbp420dPw==\n"
                    "host: bucket.s3.amazonaws.com\n"
                    "x-amz-content-sha256: 0838a79dfbddc2128d28fb4fa8d605e0a8e6d1355094000f39b6eb3feff4641f\n"
                    "x-amz-date: <redacted>\n"
                    "x-amz-security-token: <redacted>\n"
                    "*** Response Headers ***:\n"
                    "content-length: 110\n"
                    "*** Response Content ***:\n"
                    "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
                    "<Error><Code>AccessDenied</Code><Message>Access Denied</Message></Error>");

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("file missing");

                testRequestP(service, s3, HTTP_VERB_HEAD, "/BOGUS", .sseC = "rA1P");
                testResponseP(service, .code = 404);

                TEST_RESULT_BOOL(storageExistsP(s3, STRDEF("BOGUS")), false, "check");

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("info for / does not exist");

                TEST_RESULT_BOOL(storageInfoP(s3, NULL, .ignoreMissing = true).exists, false, "info for /");

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("switch to service credentials");

                hrnServerScriptClose(service);

                #define TEST_SERVICE_ROLE                           "arn:aws:iam::123456789012:role/TestRole"
                #define TEST_SERVICE_TOKEN                          "TOKEN"
                #define TEST_SERVICE_TOKEN_FILE                     TEST_PATH "/web-id-token"
                #define TEST_SERVICE_URI                                                                                           \
                    "/?Action=AssumeRoleWithWebIdentity&RoleArn=arn%3Aaws%3Aiam%3A%3A123456789012%3Arole%2FTestRole"               \
                        "&RoleSessionName=pgBackRest&Version=2011-06-15&WebIdentityToken=" TEST_SERVICE_TOKEN
                // {uncrustify_off - comment inside string}
                #define TEST_SERVICE_RESPONSE                                                                                      \
                    "<AssumeRoleWithWebIdentityResponse xmlns=\"https://sts.amazonaws.com/doc/2011-06-15/\">\n"                    \
                    "  <AssumeRoleWithWebIdentityResult>\n"                                                                        \
                    "    <Credentials>\n"                                                                                          \
                    "      <SessionToken>zz</SessionToken>\n"                                                                      \
                    "      <SecretAccessKey>yy</SecretAccessKey>\n"                                                                \
                    "      <Expiration>%s</Expiration>\n"                                                                          \
                    "      <AccessKeyId>xx</AccessKeyId>\n"                                                                        \
                    "    </Credentials>\n"                                                                                         \
                    "  </AssumeRoleWithWebIdentityResult>\n"                                                                       \
                    "</AssumeRoleWithWebIdentityResponse>"
                // {uncrustify_on}

                HRN_STORAGE_PUT_Z(storagePosixNewP(TEST_PATH_STR, .write = true), TEST_SERVICE_TOKEN_FILE, TEST_SERVICE_TOKEN);

                argList = strLstDup(commonArgList);
                hrnCfgArgRawFmt(argList, cfgOptRepoStorageHost, "%s:%u", strZ(host), testPort);
                hrnCfgArgRawStrId(argList, cfgOptRepoS3KeyType, storageS3KeyTypeWebId);
                HRN_CFG_LOAD(cfgCmdArchivePush, argList);

                TEST_ERROR(
                    storageRepoGet(0, true), OptionInvalidError,
                    "option 'repo1-s3-key-type' is 'web-id' but 'AWS_ROLE_ARN' and 'AWS_WEB_IDENTITY_TOKEN_FILE' are not set");

                setenv("AWS_ROLE_ARN", TEST_SERVICE_ROLE, true);

                TEST_ERROR(
                    storageRepoGet(0, true), OptionInvalidError,
                    "option 'repo1-s3-key-type' is 'web-id' but 'AWS_ROLE_ARN' and 'AWS_WEB_IDENTITY_TOKEN_FILE' are not set");

                setenv("AWS_WEB_IDENTITY_TOKEN_FILE", TEST_SERVICE_TOKEN_FILE, true);

                s3 = storageRepoGet(0, true);
                driver = (StorageS3 *)storageDriver(s3);

                TEST_RESULT_STR_Z(driver->credRole, TEST_SERVICE_ROLE, "check role");
                TEST_RESULT_STR_Z(driver->tokenFile, TEST_SERVICE_TOKEN_FILE, "check token file");

                // Set partSize to a small value for testing
                driver->partSize = 16;

                // Testing requires the auth http client to be redirected
                driver->credHost = hrnServerHost();
                driver->credHttpClient = httpClientNew(sckClientNew(host, testPortAuth, 5000, 5000), 5000);

                hrnServerScriptAccept(service);

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("info for missing file");

                // Get service credentials
                hrnServerScriptAccept(auth);

                testRequestP(auth, NULL, HTTP_VERB_GET, TEST_SERVICE_URI);
                testResponseP(
                    auth,
                    .content = zNewFmt(TEST_SERVICE_RESPONSE, strZ(testS3DateTime(time(NULL) + (S3_CREDENTIAL_RENEW_SEC - 1)))));

                hrnServerScriptClose(auth);

                // File missing
                testRequestP(service, s3, HTTP_VERB_HEAD, "/BOGUS", .accessKey = "xx", .securityToken = "zz");
                testResponseP(service, .code = 404);

                TEST_RESULT_BOOL(storageInfoP(s3, STRDEF("BOGUS"), .ignoreMissing = true).exists, false, "file does not exist");

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("info for file");

                // Get service credentials
                hrnServerScriptAccept(auth);

                testRequestP(auth, NULL, HTTP_VERB_GET, TEST_SERVICE_URI);
                testResponseP(
                    auth,
                    .content = zNewFmt(TEST_SERVICE_RESPONSE, strZ(testS3DateTime(time(NULL) + (S3_CREDENTIAL_RENEW_SEC * 2)))));

                hrnServerScriptClose(auth);

                testRequestP(service, s3, HTTP_VERB_HEAD, "/subdir/file1.txt");
                testResponseP(service, .header = "content-length:9999\r\nLast-Modified: Wed, 21 Oct 2015 07:28:00 GMT");

                StorageInfo info;
                TEST_ASSIGN(info, storageInfoP(s3, STRDEF("subdir/file1.txt")), "file exists");
                TEST_RESULT_BOOL(info.exists, true, "check exists");
                TEST_RESULT_UINT(info.type, storageTypeFile, "check type");
                TEST_RESULT_UINT(info.size, 9999, "check exists");
                TEST_RESULT_INT(info.timeModified, 1445412480, "check time");

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("write zero-length file (without kms)");

                testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt", .content = "");
                testResponseP(service);

                TEST_ASSIGN(write, storageNewWriteP(s3, STRDEF("file.txt")), "new write");
                TEST_RESULT_VOID(storagePutP(write, NULL), "write");

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("info check existence only");

                testRequestP(service, s3, HTTP_VERB_HEAD, "/subdir/file2.txt");
                testResponseP(service, .header = "content-length:777\r\nLast-Modified: Wed, 22 Oct 2015 07:28:00 GMT");

                TEST_ASSIGN(info, storageInfoP(s3, STRDEF("subdir/file2.txt"), .level = storageInfoLevelExists), "file exists");
                TEST_RESULT_BOOL(info.exists, true, "check exists");
                TEST_RESULT_UINT(info.type, storageTypeFile, "check type");
                TEST_RESULT_UINT(info.size, 0, "check exists");
                TEST_RESULT_INT(info.timeModified, 0, "check time");

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("errorOnMissing invalid because there are no paths");

                TEST_ERROR(
                    storageListP(s3, STRDEF("/"), .errorOnMissing = true), AssertError,
                    "assertion '!param.errorOnMissing || storageFeature(this, storageFeaturePath)' failed");

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("switch to pod-id authentication");

                hrnServerScriptClose(service);

                #define TEST_PODID_TOKEN                            "TOKEN-PODID"
                #define TEST_PODID_TOKEN_FILE                       TEST_PATH "/pod-id-token"
                #define TEST_PODID_GET                              "/v1/credentials"
                // {uncrustify_off - comment inside string}
                #define TEST_PODID_RESPONSE                                                                                        \
                    "{\n"                                                                                                          \
                    "    \"AccessKeyId\": \"gg\",\n"                                                                               \
                    "    \"SecretAccessKey\": \"hh\",\n"                                                                           \
                    "    \"Token\": \"mm\",\n"                                                                                     \
                    "    \"AccountId\":\"012345678901\",\n"                                                                        \
                    "    \"Expiration\": \"%s\"\n"                                                                                 \
                    "}"
                // {uncrustify_on}

                HRN_STORAGE_PUT_Z(storagePosixNewP(TEST_PATH_STR, .write = true), TEST_PODID_TOKEN_FILE, TEST_PODID_TOKEN);

                argList = strLstDup(commonArgList);
                hrnCfgArgRawFmt(argList, cfgOptRepoStorageHost, "%s:%u", strZ(host), testPort);
                hrnCfgArgRawStrId(argList, cfgOptRepoS3KeyType, storageS3KeyTypePodId);
                HRN_CFG_LOAD(cfgCmdArchivePush, argList);

                TEST_ERROR(
                    storageRepoGet(0, true), OptionInvalidError,
                    "option 'repo1-s3-key-type' is 'pod-id' but 'AWS_CONTAINER_CREDENTIALS_FULL_URI' and"
                    " 'AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE' are not set");

                setenv(
                    "AWS_CONTAINER_CREDENTIALS_FULL_URI", zNewFmt("http://%s:%u/v1/credentials", strZ(host), testPortAuth), true);

                TEST_ERROR(
                    storageRepoGet(0, true), OptionInvalidError,
                    "option 'repo1-s3-key-type' is 'pod-id' but 'AWS_CONTAINER_CREDENTIALS_FULL_URI' and"
                    " 'AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE' are not set");

                setenv("AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE", TEST_PODID_TOKEN_FILE, true);

                s3 = storageRepoGet(0, true);
                driver = (StorageS3 *)storageDriver(s3);

                TEST_RESULT_STR_Z(driver->tokenFile, TEST_PODID_TOKEN_FILE, "check token file");

                // Set partSize to a small value for testing
                driver->partSize = 16;

                hrnServerScriptAccept(service);

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("error without xml");

                // Get service credentials
                hrnServerScriptAccept(auth);

                testRequestP(auth, NULL, HTTP_VERB_GET, TEST_PODID_GET, .authorization = TEST_PODID_TOKEN);
                testResponseP(
                    auth,
                    .content = zNewFmt(TEST_PODID_RESPONSE, strZ(testS3DateTime(time(NULL) + (S3_CREDENTIAL_RENEW_SEC - 1)))));

                hrnServerScriptClose(auth);

                testRequestP(service, s3, HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", .accessKey = "gg", .securityToken = "mm");
                testResponseP(service, .code = 344);

                TEST_ERROR(
                    storageListP(s3, STRDEF("/")), ProtocolError,
                    "HTTP request failed with 344:\n"
                    "*** Path/Query ***:\n"
                    "GET /?delimiter=%2F&list-type=2\n"
                    "*** Request Headers ***:\n"
                    "authorization: <redacted>\n"
                    "content-length: 0\n"
                    "host: bucket." S3_TEST_HOST "\n"
                    "x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n"
                    "x-amz-date: <redacted>\n"
                    "x-amz-security-token: <redacted>");

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("error with xml");

                // Get service credentials
                hrnServerScriptAccept(auth);

                testRequestP(auth, NULL, HTTP_VERB_GET, TEST_PODID_GET, .authorization = TEST_PODID_TOKEN);
                testResponseP(
                    auth,
                    .content = zNewFmt(TEST_PODID_RESPONSE, strZ(testS3DateTime(time(NULL) + (S3_CREDENTIAL_RENEW_SEC * 2)))));

                hrnServerScriptClose(auth);

                testRequestP(service, s3, HTTP_VERB_GET, "/?delimiter=%2F&list-type=2");
                testResponseP(
                    service, .code = 344,
                    .content =
                        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
                        "<Error>"
                        "<Code>SomeOtherCode</Code>"
                        "</Error>");

                TEST_ERROR(
                    storageListP(s3, STRDEF("/")), ProtocolError,
                    "HTTP request failed with 344:\n"
                    "*** Path/Query ***:\n"
                    "GET /?delimiter=%2F&list-type=2\n"
                    "*** Request Headers ***:\n"
                    "authorization: <redacted>\n"
                    "content-length: 0\n"
                    "host: bucket." S3_TEST_HOST "\n"
                    "x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n"
                    "x-amz-date: <redacted>\n"
                    "x-amz-security-token: <redacted>\n"
                    "*** Response Headers ***:\n"
                    "content-length: 79\n"
                    "*** Response Content ***:\n"
                    "<?xml version=\"1.0\" encoding=\"UTF-8\"?><Error><Code>SomeOtherCode</Code></Error>");

                // Auth service no longer needed
                hrnServerScriptEnd(auth);

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("list basic level");

                testRequestP(service, s3, HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=path%2Fto%2F");
                testResponseP(
                    service,
                    .content =
                        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
                        "<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
                        "    <IsTruncated>false</IsTruncated>"
                        "    <Contents>"
                        "        <Key>path/to/test_file</Key>"
                        "        <LastModified>2009-10-12T17:50:30.000Z</LastModified>"
                        "        <Size>787</Size>"
                        "    </Contents>"
                        "   <CommonPrefixes>"
                        "       <Prefix>path/to/test_path/</Prefix>"
                        "   </CommonPrefixes>"
                        "</ListBucketResult>");

                TEST_ERROR(
                    storageNewItrP(s3, STRDEF("/"), .errorOnMissing = true), AssertError,
                    "assertion '!param.errorOnMissing || storageFeature(this, storageFeaturePath)' failed");

                TEST_STORAGE_LIST(
                    s3, "/path/to",
                    "test_file {s=787, t=1255369830}\n"
                    "test_path/\n",
                    .level = storageInfoLevelBasic, .noRecurse = true);

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("list exists level");

                testRequestP(service, s3, HTTP_VERB_GET, "/?delimiter=%2F&list-type=2");
                testResponseP(
                    service,
                    .content =
                        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
                        "<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
                        "    <IsTruncated>false</IsTruncated>"
                        "    <Contents>"
                        "        <Key>test1.txt</Key>"
                        "    </Contents>"
                        "   <CommonPrefixes>"
                        "       <Prefix>path1/</Prefix>"
                        "   </CommonPrefixes>"
                        "</ListBucketResult>");

                TEST_STORAGE_LIST(
                    s3, "/",
                    "path1/\n"
                    "test1.txt\n",
                    .noRecurse = true);

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("list a file in root with expression");

                testRequestP(service, s3, HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=test");
                testResponseP(
                    service,
                    .content =
                        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
                        "<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
                        "    <IsTruncated>false</IsTruncated>"
                        "    <Contents>"
                        "        <Key>test1.txt</Key>"
                        "    </Contents>"
                        "</ListBucketResult>");

                TEST_STORAGE_LIST(
                    s3, "/",
                    "test1.txt\n",
                    .noRecurse = true, .expression = "^test.*$");

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("list files with continuation");

                testRequestP(service, s3, HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=path%2Fto%2F");
                testResponseP(
                    service,
                    .content =
                        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
                        "<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
                        "    <IsTruncated>true</IsTruncated>"
                        "    <NextContinuationToken>1ueGcxLPRx1Tr/XYExHnhbYLgveDs2J/wm36Hy4vbOwM=</NextContinuationToken>"
                        "    <Contents>"
                        "        <Key>path/to/test1.txt</Key>"
                        "    </Contents>"
                        "    <Contents>"
                        "        <Key>path/to/test2.txt</Key>"
                        "    </Contents>"
                        "   <CommonPrefixes>"
                        "       <Prefix>path/to/path1/</Prefix>"
                        "   </CommonPrefixes>"
                        "</ListBucketResult>");

                testRequestP(
                    service, s3, HTTP_VERB_GET,
                    "/?continuation-token=1ueGcxLPRx1Tr%2FXYExHnhbYLgveDs2J%2Fwm36Hy4vbOwM%3D&delimiter=%2F&list-type=2"
                    "&prefix=path%2Fto%2F");
                testResponseP(
                    service,
                    .content =
                        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
                        "<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
                        "    <IsTruncated>false</IsTruncated>"
                        "    <Contents>"
                        "        <Key>path/to/test3.txt</Key>"
                        "    </Contents>"
                        "   <CommonPrefixes>"
                        "       <Prefix>path/to/path2/</Prefix>"
                        "   </CommonPrefixes>"
                        "</ListBucketResult>");

                TEST_STORAGE_LIST(
                    s3, "/path/to",
                    "path1/\n"
                    "path2/\n"
                    "test1.txt\n"
                    "test2.txt\n"
                    "test3.txt\n",
                    .noRecurse = true);

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("list files with expression");

                testRequestP(service, s3, HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=path%2Fto%2Ftest");
                testResponseP(
                    service,
                    .content =
                        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
                        "<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
                        "    <IsTruncated>false</IsTruncated>"
                        "    <Contents>"
                        "        <Key>path/to/test1.txt</Key>"
                        "    </Contents>"
                        "    <Contents>"
                        "        <Key>path/to/test2.txt</Key>"
                        "    </Contents>"
                        "    <Contents>"
                        "        <Key>path/to/test3.txt</Key>"
                        "    </Contents>"
                        "   <CommonPrefixes>"
                        "       <Prefix>path/to/test1.path/</Prefix>"
                        "   </CommonPrefixes>"
                        "   <CommonPrefixes>"
                        "       <Prefix>path/to/test2.path/</Prefix>"
                        "   </CommonPrefixes>"
                        "</ListBucketResult>");

                TEST_STORAGE_LIST(
                    s3, "/path/to",
                    "test1.path\n"
                    "test1.txt\n"
                    "test3.txt\n",
                    .level = storageInfoLevelExists, .noRecurse = true, .expression = "^test(1|3)");

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("switch to path-style URIs");

                hrnServerScriptClose(service);

                argList = strLstDup(commonArgList);
                hrnCfgArgRawStrId(argList, cfgOptRepoS3UriStyle, storageS3UriStylePath);
                hrnCfgArgRawFmt(argList, cfgOptRepoStorageHost, "https://%s", strZ(host));
                hrnCfgArgRawFmt(argList, cfgOptRepoStoragePort, "%u", testPort);
                hrnCfgEnvRemoveRaw(cfgOptRepoS3Token);
                HRN_CFG_LOAD(cfgCmdArchivePush, argList);

                s3 = storageRepoGet(0, true);
                driver = (StorageS3 *)storageDriver(s3);

                // Set deleteMax to a small value for testing
                driver->deleteMax = 2;

                hrnServerScriptAccept(service);

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("error when no recurse because there are no paths");

                TEST_ERROR(
                    storagePathRemoveP(s3, STRDEF("/")), AssertError,
                    "assertion 'param.recurse || storageFeature(this, storageFeaturePath)' failed");

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("remove files from root");

                testRequestP(service, s3, HTTP_VERB_GET, "/bucket/?list-type=2");
                testResponseP(
                    service,
                    .content =
                        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
                        "<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
                        "    <IsTruncated>false</IsTruncated>"
                        "    <Contents>"
                        "        <Key>test1.txt</Key>"
                        "    </Contents>"
                        "    <Contents>"
                        "        <Key>path1/xxx.zzz</Key>"
                        "    </Contents>"
                        "</ListBucketResult>");

                testRequestP(
                    service, s3, HTTP_VERB_POST, "/bucket/?delete=",
                    .content =
                        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
                        "<Delete><Quiet>true</Quiet>"
                        "<Object><Key>test1.txt</Key></Object>"
                        "<Object><Key>path1/xxx.zzz</Key></Object>"
                        "</Delete>\n");
                testResponseP(
                    service, .content = "<DeleteResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"></DeleteResult>");

                TEST_RESULT_VOID(storagePathRemoveP(s3, STRDEF("/"), .recurse = true), "remove");

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("remove files in empty subpath (nothing to do)");

                testRequestP(service, s3, HTTP_VERB_GET, "/bucket/?list-type=2&prefix=path%2F");
                testResponseP(
                    service,
                    .content =
                        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
                        "<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
                        "    <IsTruncated>false</IsTruncated>"
                        "</ListBucketResult>");

                TEST_RESULT_VOID(storagePathRemoveP(s3, STRDEF("/path"), .recurse = true), "remove");

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("remove files with continuation");

                testRequestP(service, s3, HTTP_VERB_GET, "/bucket/?list-type=2&prefix=path%2Fto%2F");
                testResponseP(
                    service,
                    .content =
                        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
                        "<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
                        "    <IsTruncated>true</IsTruncated>"
                        "    <NextContinuationToken>continue</NextContinuationToken>"
                        "   <CommonPrefixes>"
                        "       <Prefix>path/to/test3/</Prefix>"
                        "   </CommonPrefixes>"
                        "    <Contents>"
                        "        <Key>path/to/test1.txt</Key>"
                        "    </Contents>"
                        "</ListBucketResult>");

                testRequestP(service, s3, HTTP_VERB_GET, "/bucket/?continuation-token=continue&list-type=2&prefix=path%2Fto%2F");
                testResponseP(
                    service,
                    .content =
                        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
                        "<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
                        "    <IsTruncated>false</IsTruncated>"
                        "    <Contents>"
                        "        <Key>path/to/test3.txt</Key>"
                        "    </Contents>"
                        "    <Contents>"
                        "        <Key>path/to/test2.txt</Key>"
                        "    </Contents>"
                        "</ListBucketResult>");

                testRequestP(
                    service, s3, HTTP_VERB_POST, "/bucket/?delete=",
                    .content =
                        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
                        "<Delete><Quiet>true</Quiet>"
                        "<Object><Key>path/to/test1.txt</Key></Object>"
                        "<Object><Key>path/to/test3.txt</Key></Object>"
                        "</Delete>\n");
                testResponseP(service);

                testRequestP(
                    service, s3, HTTP_VERB_POST, "/bucket/?delete=",
                    .content =
                        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
                        "<Delete><Quiet>true</Quiet>"
                        "<Object><Key>path/to/test2.txt</Key></Object>"
                        "</Delete>\n");
                testResponseP(service);

                TEST_RESULT_VOID(storagePathRemoveP(s3, STRDEF("/path/to"), .recurse = true), "remove");

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("path remove error and retry");

                testRequestP(service, s3, HTTP_VERB_GET, "/bucket/?list-type=2&prefix=path%2F");
                testResponseP(
                    service,
                    .content =
                        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
                        "<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
                        "    <IsTruncated>false</IsTruncated>"
                        "    <Contents>"
                        "        <Key>path/sample.txt</Key>"
                        "    </Contents>"
                        "    <Contents>"
                        "        <Key>path/sample2.txt</Key>"
                        "    </Contents>"
                        "</ListBucketResult>");

                testRequestP(
                    service, s3, HTTP_VERB_POST, "/bucket/?delete=",
                    .content =
                        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
                        "<Delete><Quiet>true</Quiet>"
                        "<Object><Key>path/sample.txt</Key></Object>"
                        "<Object><Key>path/sample2.txt</Key></Object>"
                        "</Delete>\n");
                testResponseP(
                    service,
                    .content =
                        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
                        "<DeleteResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
                        "<Error><Key>path/sample2.txt</Key><Code>AccessDenied</Code><Message>Access Denied</Message></Error>"
                        "</DeleteResult>");

                testRequestP(service, s3, HTTP_VERB_DELETE, "/bucket/path/sample2.txt");
                testResponseP(service, .code = 204);

                TEST_RESULT_VOID(storagePathRemoveP(s3, STRDEF("/path"), .recurse = true), "remove path");

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("remove file");

                testRequestP(service, s3, HTTP_VERB_DELETE, "/bucket/path/to/test.txt");
                testResponseP(service, .code = 204);

                TEST_RESULT_VOID(storageRemoveP(s3, STRDEF("/path/to/test.txt")), "remove");

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("switch to time limited");

                hrnServerScriptClose(service);

                argList = strLstDup(commonArgList);
                hrnCfgArgRawFmt(argList, cfgOptRepoStorageHost, "%s:%u", strZ(host), testPort);
                hrnCfgArgRawZ(argList, cfgOptPgPath, "/pg1");
                hrnCfgArgRawZ(argList, cfgOptRepo, "1");
                hrnCfgArgRawZ(argList, cfgOptRepoTargetTime, "2024-08-04 02:54:09+00");
                hrnCfgArgRawBool(argList, cfgOptRepoS3RequesterPays, true);
                HRN_CFG_LOAD(cfgCmdArchiveGet, argList);

                s3 = storageRepoGet(0, false);

                hrnServerScriptAccept(service);

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("list with time limit");

                testRequestP(service, s3, HTTP_VERB_GET, "/?delimiter=%2F&prefix=path%2Fto%2F&versions=", .requesterPays = true);
                testResponseP(
                    service,
                    .content =
                        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
                        "<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
                        "    <IsTruncated>true</IsTruncated>"
                        "    <NextContinuationToken>1ueG</NextContinuationToken>"
                        "    <DeleteMarker>"
                        "        <Key>path/to/test_file</Key>"
                        "        <LastModified>2024-08-04T02:54:10.000Z</LastModified>"
                        "    </DeleteMarker>"
                        "    <Version>"
                        "        <Key>path/to/test_file</Key>"
                        "        <LastModified>2024-08-04T02:54:10.000Z</LastModified>"
                        "    </Version>"
                        "</ListBucketResult>");

                testRequestP(
                    service, s3, HTTP_VERB_GET, "/?continuation-token=1ueG&delimiter=%2F&prefix=path%2Fto%2F&versions=",
                    .requesterPays = true);
                testResponseP(
                    service,
                    .content =
                        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
                        "<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
                        "    <IsTruncated>false</IsTruncated>"
                        "    <Version>"
                        "        <Key>path/to/test_file</Key>"
                        "        <LastModified>2024-08-04T02:54:09.000Z</LastModified>"
                        "        <Size>737</Size>"
                        "        <VersionId>bbbb</VersionId>"
                        "    </Version>"
                        "    <Version>"
                        "        <Key>path/to/test_file</Key>"
                        "        <LastModified>2024-08-04T02:54:09.000Z</LastModified>"
                        "    </Version>"
                        "    <Version>"
                        "        <Key>path/to/test_file2</Key>"
                        "        <LastModified>2024-08-04T02:54:10.000Z</LastModified>"
                        "    </Version>"
                        "   <CommonPrefixes>"
                        "       <Prefix>path/to/test_path/</Prefix>"
                        "   </CommonPrefixes>"
                        "</ListBucketResult>");

                TEST_STORAGE_LIST(
                    s3, "/path/to",
                    "test_file {s=737, t=1722740049, v=bbbb}\n"
                    "test_path/\n",
                    .level = storageInfoLevelBasic, .noRecurse = true);

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("get file with time limit");

                testRequestP(service, s3, HTTP_VERB_GET, "/?delimiter=%2F&prefix=path%2F3%2F&versions=", .requesterPays = true);
                testResponseP(
                    service,
                    .content =
                        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
                        "<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
                        "    <IsTruncated>false</IsTruncated>"
                        "    <Version>"
                        "        <Key>path/3/test_file</Key>"
                        "        <LastModified>2024-08-04T02:54:10.000Z</LastModified>"
                        "    </Version>"
                        "    <Version>"
                        "        <Key>path/3/test_file</Key>"
                        "        <LastModified>2024-08-04T02:54:09.000Z</LastModified>"
                        "        <Size>6</Size>"
                        "        <VersionId>bbbb</VersionId>"
                        "    </Version>"
                        "   <CommonPrefixes>"
                        "       <Prefix>path/3/test_path/</Prefix>"
                        "   </CommonPrefixes>"
                        "</ListBucketResult>");

                testRequestP(service, s3, HTTP_VERB_GET, "/path/3/test_file?versionId=bbbb", .requesterPays = true);
                testResponseP(service, .content = "123456");

                TEST_RESULT_STR_Z(
                    strNewBuf(storageGetP(storageNewReadP(s3, STRDEF("/path/3/test_file")))), "123456", "get file");

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("get missing file with time limit");

                testRequestP(service, s3, HTTP_VERB_GET, "/?delimiter=%2F&versions=", .requesterPays = true);
                testResponseP(
                    service,
                    .content =
                        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
                        "<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
                        "    <IsTruncated>false</IsTruncated>"
                        "    <Version>"
                        "        <Key>missing_file</Key>"
                        "        <LastModified>2024-08-04T02:54:10.000Z</LastModified>"
                        "    </Version>"
                        "    <DeleteMarker>"
                        "        <Key>missing_file</Key>"
                        "        <LastModified>2024-08-04T02:54:09.000Z</LastModified>"
                        "    </DeleteMarker>"
                        "    <CommonPrefixes>"
                        "       <Prefix>test_path/</Prefix>"
                        "    </CommonPrefixes>"
                        "</ListBucketResult>");

                TEST_RESULT_PTR(
                    storageGetP(storageNewReadP(s3, STRDEF("missing_file"), .ignoreMissing = true)), NULL, "missing file");

                // -----------------------------------------------------------------------------------------------------------------
                hrnServerScriptEnd(service);
            }
            HRN_FORK_PARENT_END();
        }
        HRN_FORK_END();
    }

    // *****************************************************************************************************************************
    if (testBegin("StorageS3, StorageReadS3, and StorageWriteS3 with HTTP"))
    {
        // -------------------------------------------------------------------------------------------------------------------------
        HRN_FORK_BEGIN()
        {
            const unsigned int testPort = hrnServerPortNext();

            HRN_FORK_CHILD_BEGIN(.prefix = "s3 http server", .timeout = 5000)
            {
                TEST_RESULT_VOID(
                    hrnServerRunP(HRN_FORK_CHILD_READ(), hrnServerProtocolSocket, testPort),
                    "s3 http server");
            }
            HRN_FORK_CHILD_END();

            HRN_FORK_PARENT_BEGIN()
            {
                IoWrite *service = hrnServerScriptBegin(
                    ioFdWriteNewOpen(STRDEF("s3 http client"), HRN_FORK_PARENT_WRITE_FD(0), 2000));

                StringList *argList = strLstDup(commonArgList);
                hrnCfgArgRawFmt(argList, cfgOptRepoStorageHost, "http://%s:%u", strZ(host), testPort);
                HRN_CFG_LOAD(cfgCmdArchivePush, argList);

                Storage *s3 = storageRepoGet(0, true);
                StorageS3 *driver = (StorageS3 *)storageDriver(s3);

                // Set partSize to a small value for testing
                driver->partSize = 16;

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("error on missing file");

                hrnServerScriptAccept(service);
                testRequestP(service, s3, HTTP_VERB_GET, "/file.txt");
                testResponseP(service, .code = 404);

                TEST_ERROR(
                    storageGetP(storageNewReadP(s3, STRDEF("file.txt"))), FileMissingError,
                    "unable to open missing file '/file.txt' for read");

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("get file with offset and limit");

                testRequestP(service, s3, HTTP_VERB_GET, "/file.txt", .range = "1-21");
                testResponseP(service, .content = "this is a sample file");

                TEST_RESULT_STR_Z(
                    strNewBuf(storageGetP(storageNewReadP(s3, STRDEF("file.txt"), .offset = 1, .limit = VARUINT64(21)))),
                    "this is a sample file", "get file");

                // -----------------------------------------------------------------------------------------------------------------
                TEST_TITLE("write file in chunks with something left over on close");

                testRequestP(service, s3, HTTP_VERB_POST, "/file.txt?uploads=");
                testResponseP(
                    service,
                    .content =
                        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
                        "<InitiateMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
                        "<Bucket>bucket</Bucket>"
                        "<Key>file.txt</Key>"
                        "<UploadId>RR55</UploadId>"
                        "</InitiateMultipartUploadResult>");

                testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=1&uploadId=RR55", .content = "1234567890123456");
                testResponseP(service, .header = "etag:RR551");

                testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=2&uploadId=RR55", .content = "7890");
                testResponseP(service, .header = "eTag:RR552");

                testRequestP(
                    service, s3, HTTP_VERB_POST, "/file.txt?uploadId=RR55",
                    .content =
                        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
                        "<CompleteMultipartUpload>"
                        "<Part><PartNumber>1</PartNumber><ETag>RR551</ETag></Part>"
                        "<Part><PartNumber>2</PartNumber><ETag>RR552</ETag></Part>"
                        "</CompleteMultipartUpload>\n");
                testResponseP(
                    service,
                    .content =
                        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
                        "<CompleteMultipartUploadResult><ETag>XXX</ETag></CompleteMultipartUploadResult>");

                // Check that block size is updated during write
                const size_t ioBufferSizeDefault = ioBufferSize();
                ioBufferSizeSet(6);

                StorageWrite *write = NULL;
                TEST_ASSIGN(write, storageNewWriteP(s3, STRDEF("file.txt")), "new write");

                ioWriteOpen(storageWriteIo(write));
                ioWrite(storageWriteIo(write), BUFSTRDEF("123456789012345678"));

                TEST_RESULT_VOID(
                    bufResize(((StorageWriteS3 *)ioWriteDriver(storageWriteIo(write)))->partBuffer, 17),
                    "resize part buffer to 17");

                ioWrite(storageWriteIo(write), BUFSTRDEF("90"));

                TEST_RESULT_UINT(
                    ((StorageWriteS3 *)ioWriteDriver(storageWriteIo(write)))->partSize, 16, "part buffer reset to 16 (default)");

                ioWriteClose(storageWriteIo(write));
                ioBufferSizeSet(ioBufferSizeDefault);

                hrnServerScriptEnd(service);
            }
            HRN_FORK_PARENT_END();
        }
        HRN_FORK_END();
    }

    FUNCTION_HARNESS_RETURN_VOID();
}
