/** @file databundle.cpp  Classic data files: PK3, WAD, LMP, DED, DEH.
 *
 * @authors Copyright (c) 2016-2017 Jaakko Keränen <jaakko.keranen@iki.fi>
 *
 * @par License
 * GPL: http://www.gnu.org/licenses/gpl.html
 *
 * <small>This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by the
 * Free Software Foundation; either version 2 of the License, or (at your
 * option) any later version. This program is distributed in the hope that it
 * will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
 * Public License for more details. You should have received a copy of the GNU
 * General Public License along with this program; if not, see:
 * http://www.gnu.org/licenses</small>
 */

#include "doomsday/resource/databundle.h"
#include "doomsday/filesys/datafolder.h"
#include "doomsday/filesys/datafile.h"
#include "doomsday/resource/bundles.h"
#include "doomsday/resource/resources.h"
#include "doomsday/resource/lumpdirectory.h"
#include "doomsday/doomsdayapp.h"
#include "doomsday/Games"

#include <de/charsymbols.h>
#include <de/App>
#include <de/ArchiveFeed>
#include <de/Info>
#include <de/LinkFile>
#include <de/LogBuffer>
#include <de/MetadataBank>
#include <de/Package>
#include <de/PackageLoader>
#include <de/Path>
#include <de/TextValue>
#include <QRegExp>
#include <QRegularExpression>
#include <QTextCodec>

using namespace de;

static String const VAR_PATH        ("path");
static String const VAR_VERSION     ("version");
static String const VAR_LICENSE     ("license");
static String const VAR_AUTHOR      ("author");
static String const VAR_TITLE       ("title");
static String const VAR_TAGS        ("tags");
static String const VAR_NOTES       ("notes");
static String const VAR_DATA_FILES  ("dataFiles");
static String const VAR_BUNDLE_SCORE("bundleScore");
static String const VAR_REQUIRES    ("requires");
static String const VAR_RECOMMENDS  ("recommends");
static String const VAR_EXTRAS      ("extras");
static String const VAR_CATEGORY    ("category");

static String const CACHE_CATEGORY  ("DataBundle");

namespace internal
{
    static char const *formatDescriptions[] =
    {
        "unknown",
        "PK3 archive",
        "WAD file",
        "IWAD file",
        "PWAD file",
        "data lump",
        "Doomsday Engine definitions",
        "DeHackEd patch", // importdeh plugin
        "collection"
    };
}

DENG2_PIMPL(DataBundle), public Lockable
{
    bool ignored = false;
    SafePtr<File> source;
    Format format;
    String packageId; // linked under /sys/bundles/
    String versionedPackageId;
    std::unique_ptr<res::LumpDirectory> lumpDir;
    SafePtr<LinkFile> pkgLink;

    Impl(Public *i, Format fmt) : Base(i), format(fmt)
    {}

    ~Impl()
    {
        DENG2_GUARD(this);
        delete pkgLink.get();
    }

    static Folder &bundleFolder()
    {
        return App::rootFolder().locate<Folder>(QStringLiteral("/sys/bundles"));
    }

    /**
     * Identifies the data bundle and sets up a package link under "/sys/bundles" with
     * the appropriate metadata.
     *
     * Sets up the package metadata according to the best matched known information or
     * autogenerated entries.
     *
     * @return @c true, if the bundle was identified; otherwise @c false.
     */
    bool identify()
    {
        DENG2_GUARD(this);

        // It is sufficient to identify each bundle only once.
        if (ignored || !packageId.isEmpty()) return false;

        // Load the lump directory of WAD files.
        if (format == Wad || format == Pwad || format == Iwad)
        {
            // The lump directory needs to be loaded before matching against known
            // bundles because it can be used for identification.
            lumpDir.reset(new res::LumpDirectory(source->as<ByteArrayFile>()));
            if (!lumpDir->isValid())
            {
                throw FormatError("DataBundle::identify",
                                  dynamic_cast<File *>(thisPublic)->description() +
                                  ": file contents may be corrupted " DENG2_CHAR_MDASH
                                  " WAD lump directory was not found");
            }

            // Determine the WAD type, if unspecified.
            format = (lumpDir->type() == res::LumpDirectory::Pwad? Pwad : Iwad);

            /*
            qDebug() << self().description()
                     << "format:" << (lumpDir->type()==res::LumpDirectory::Pwad? "PWAD" : "IWAD")
                     << "\nfileName:" << source->name()
                     << "\nfileSize:" << source->size()
                     << "\nlumpDirCRC32:" << QString::number(lumpDir->crc32(), 16).toLatin1();
            */
        }
        else if (!self().containerPackageId().isEmpty())
        {
            // This file is inside a package, so the package will take care of it.
            /*qDebug() << "[DataBundle]" << source->description().toLatin1().constData()
                     << "is nested, no package will be generated";*/
            ignored = true;
            return false;
        }

        if (isAutoLoaded())
        {
            // We're still loading with FS1, so it will handle auto-loaded files.
            ignored = true;
            return false;
        }

        DataBundle *container = self().containerBundle();
        if (container)
        {
            // Make sure that the container has been fully identified.
            container->identifyPackages();

            if (format == Ded && container->format() == Pk3)
            {
                // DED files are typically explicitly imported from some main DED file
                // in the container.
                ignored = true;
                return false;
            }
#if 0
            if (container->isLinkedAsPackage() &&
                container->format() != Collection &&
                isAutoLoaded())
            {
                qDebug() << container->d->versionedPackageId
                         << "loads data file"
                         << dataFilePath;

                // No package will be generated for this file. It will be loaded via
                // the container package.
                container->packageMetadata()[VAR_DATA_FILES]
                        .value<ArrayValue>().add(new TextValue(dataFilePath));
                //ignored = true;
                //return;
            }
#endif

            if (container->d->ignored)
            {
                ignored = true; // No package for this.
                return false;
            }
        }

        Record const meta = cachedMetadata();
        packageId = meta.gets(Package::VAR_ID);
        versionedPackageId = packageId;

        // Finally, make a link that represents the package.
        if (auto chosen = chooseUniqueLinkPathAndVersion(self().asFile(), packageId,
                                                         meta.gets(VAR_VERSION),
                                                         meta.geti(VAR_BUNDLE_SCORE)))
        {
            LOGDEV_RES_VERBOSE("Linking %s as %s") << self().asFile().path() << chosen.path;

            //qDebug() << "linking" << self().asFile().path() << chosen.path;

            pkgLink.reset(&bundleFolder().add(LinkFile::newLinkToFile(self().asFile(), chosen.path)));

            // Set up package metadata in the link.
            Record &metadata = Package::initializeMetadata(*pkgLink, packageId);
            metadata.copyMembersFrom(meta);
            metadata.set(VAR_VERSION, !chosen.version.isEmpty()? chosen.version : "0.0");

            // Compose a versioned ID.
            if (!chosen.version.isEmpty())
            {
                versionedPackageId += "_" + chosen.version;
            }

            LOG_RES_VERBOSE("Generated package:\n%s") << metadata.asText();

            App::fileSystem().index(*pkgLink);

            // Make this a required package in the container bundle.
            if (container &&
                container->isLinkedAsPackage() &&
                container->format() == Collection)
            {
                //File &containerFile = *container->d->pkgLink;

                String subset = VAR_RECOMMENDS;
                String parentFolder = self().asFile().path().fileNamePath().fileName();
                if (!parentFolder.compareWithoutCase(QStringLiteral("Extra")))
                {
                    subset = VAR_EXTRAS;
                }
                else if (!parentFolder.compareWithoutCase(QStringLiteral("Required")))
                {
                    subset = VAR_REQUIRES;
                }
                container->packageMetadata().insertToSortedArray(subset, new TextValue(versionedPackageId));

                /*
                LOGDEV_RES_VERBOSE("%s ") << container->d->versionedPackageId
                         << "(" << container->d->pkgLink->objectNamespace().gets("package.tags", "") << ") "
                         << subset << " "
                         << versionedPackageId
                         << " (" << metadata.gets("tags", "") << ") from "
                         << self().asFile().path();
                */
                //Package::addRequiredPackage(containerFile, versionedPackageId);
            }
            return true;
        }
        else
        {
            ignored = true;
            return false;
        }
    }

    /**
     * Fetches cached data bundle metadata from the metadata bank, or rebuilds the
     * metadata if the cached data is missing or invalid. Updated metadata is saved
     * in the metadata bank.
     *
     * @return Bundle metadata.
     */
    Record cachedMetadata()
    {
        Record meta;
        Block metaId = self().asFile().metaId();

        // Include container in the meta ID.
        if (auto *container = self().containerBundle())
        {
            metaId = Block(metaId + container->asFile().metaId()).md5Hash();
        }

        try
        {
            // Maybe we already have this?
            if (Block cached = MetadataBank::get().check(CACHE_CATEGORY, metaId))
            {
                // Well, our work here has already been done.
                cached = cached.decompressed();
                Reader(cached).withHeader() >> meta;
                return meta;
            }
        }
        catch (Error const &er)
        {
            LOGDEV_RES_WARNING("Corrupt cached metadata: %s") << er.asText();
        }

        meta = buildMetadata();

        // Now we can put it in the cache.
        {
            Block buf;
            Writer(buf).withHeader() << meta;
            MetadataBank::get().setMetadata(CACHE_CATEGORY, metaId, buf.compressed());
        }

        return meta;
    }

    Record buildMetadata()
    {
        const String dataFilePath = self().asFile().path();
        const auto * container    = self().containerBundle();

        // Search for known data files in the bundle registry.
        res::Bundles::MatchResult matched = DoomsdayApp::bundles().match(self());

        // Metadata for the package will be collected into this record.
        Record meta;
        meta.set(VAR_PATH,         dataFilePath);
        meta.set(VAR_BUNDLE_SCORE, matched.bestScore);

        if (format != Collection)
        {
            // Classic data files are loaded via the "dataFiles" array (in the listed order).
            // However, collections are represented directly as Doomsday packages.
            // Paths in "dataFiles" are relative to the package root.
            meta.addArray(VAR_DATA_FILES, new ArrayValue({ new TextValue(dataFilePath.fileName()) }));
        }
        else
        {
            meta.addArray(VAR_DATA_FILES);

            // Collections have a number of subsets.
            meta.addArray(VAR_REQUIRES);
            meta.addArray(VAR_RECOMMENDS);
            meta.addArray(VAR_EXTRAS);
        }

        // At least two criteria must match -- otherwise simply having the correct
        // type would be accepted.
        if (matched)
        {
            // Package metadata has been defined for this file (databundles.dei).
            packageId = matched.packageId;

            if (lumpDir)
            {
                meta.set(QStringLiteral("lumpDirCRC32"), lumpDir->crc32())
                        .value<NumberValue>().setSemanticHints(NumberValue::Hex);
            }

            meta.set(Package::VAR_TITLE, matched.bestMatch->keyValue("info:title"));
            meta.set(VAR_VERSION, matched.packageVersion.fullNumber());
            meta.set(VAR_AUTHOR,  matched.bestMatch->keyValue("info:author"));
            meta.set(VAR_LICENSE, matched.bestMatch->keyValue("info:license", "Unknown"));
            meta.set(VAR_TAGS,    matched.bestMatch->keyValue("info:tags"));
        }
        else
        {
            meta.set(Package::VAR_TITLE, self().asFile().name());
            meta.set(VAR_AUTHOR,  "Unknown");
            meta.set(VAR_LICENSE, "Unknown");
            meta.set(VAR_TAGS,    (format == Iwad || format == Pwad)? ".wad" : "");

            // Generate a default identifier based on the information we have.
            static String const formatDomains[] = {
                "file.local",
                "file.pk3",
                "file.wad",
                "file.iwad",
                "file.pwad",
                "file.lmp",
                "file.ded",
                "file.deh",
                "file.box"
            };

            // Containers become part of the identifier.
            for (const DataBundle *c = container; c; c = c->containerBundle())
            {
                String containedId = cleanIdentifier(
                    stripVersion(c->sourceFile().name().fileNameWithoutExtension()));

                // Additionally include the parent subfolder within the container into the ID.
                if (c == container && dataFilePath.fileNamePath() != c->asFile().path())
                {
                    containedId = containedId.concatenateMember(cleanIdentifier(
                        stripVersion(dataFilePath.fileNamePath().fileNameWithoutExtension())));
                }

                packageId = containedId.concatenateMember(packageId);
            }

            // The file name may contain a version number.
            Version parsedVersion("");
            String strippedName = stripVersion(source->name().fileNameWithoutExtension(),
                                               &parsedVersion);
            if (strippedName != source->name() && parsedVersion.isValid())
            {
                meta.set(VAR_VERSION, parsedVersion.fullNumber());
            }
            else
            {
                // Compose a default version number from file status.
                meta.set(VAR_VERSION, self().asFile().status().modifiedAt
                         .asDateTime().toString("0.yyyy.MMdd.hhmm"));
            }

            packageId = stripRedundantParts(formatDomains[format]
                                            .concatenateMember(packageId)
                                            .concatenateMember(cleanIdentifier(strippedName)));

            auto &root = App::rootFolder();

            // WAD files sometimes come with a matching TXT file.
            checkAuxiliaryNotes(meta);

            // There may be Snowberry metadata available:
            // - Info entry inside root folder
            // - .manifest companion
            File const *sbInfo = root.tryLocate<File const>(
                        dataFilePath.fileNamePath() / dataFilePath.fileNameWithoutExtension() +
                        ".manifest");
            if (!sbInfo)
            {
                // Check the Snowberry-style ID.
                String fn = dataFilePath.fileName();
                if (int dotPos = fn.lastIndexOf('.'))
                {
                    if (dotPos >= 0) fn[dotPos] = '-';
                }
                sbInfo = root.tryLocate<File const>(dataFilePath.fileNamePath()/fn + ".manifest");
            }
            if (!sbInfo)
            {
                sbInfo = root.tryLocate<File const>(dataFilePath/"Info");
            }
            if (!sbInfo)
            {
                sbInfo = root.tryLocate<File const>(dataFilePath/"Contents/Info");
            }
            if (sbInfo)
            {
                parseSnowberryInfo(*sbInfo, meta);
            }
        }

        meta.set("ID", packageId);

        parseNotesForMetadata(meta);

        if (container && container->format() == Collection)
        {
            // Box contents are normally hidden.
            meta.appendUniqueWord(VAR_TAGS, "hidden");
        }

        // Check for built-in tags.
        {
            // Cached copies of remote files.
            if (dataFilePath.startsWith("/home/cache/remote/"))
            {
                meta.appendUniqueWord(VAR_TAGS, "hidden");
                meta.appendUniqueWord(VAR_TAGS, "cached");
            }

            // Master Levels of Doom.
            {
                static const struct {
                    uint32_t    crc32;
                    const char *filename;
                } masterLevels[] = {{0xaa78f088, "attack.wad"},   {0x56bf62c2, "blacktwr.wad"},
                                    {0xa54aee5b, "bloodsea.wad"}, {0x5a8fb0f5, "canyon.wad"},
                                    {0x20954e50, "catwalk.wad"},  {0xb237de09, "combine.wad"},
                                    {0x9f051374, "fistula.wad"},  {0x86491354, "garrison.wad"},
                                    {0x60cc2385, "geryon.wad"},   {0x9755324e, "manor.wad"},
                                    {0xcfe7d641, "mephisto.wad"}, {0xc400cf65, "minos.wad"},
                                    {0x89386748, "nessus.wad"},   {0xac8808e9, "paradox.wad"},
                                    {0x8a84cc17, "subspace.wad"}, {0x9ffd4024, "subterra.wad"},
                                    {0x96919f5e, "teeth.wad"},    {0xd8d46a55, "ttrap.wad"},
                                    {0x1726dbb7, "vesperas.wad"}, {0xd421fe9d, "virgil.wad"}};
                if (format == Pwad)
                {
                    for (const auto &spec : masterLevels)
                    {
                        if (lumpDir->crc32() == spec.crc32 &&
                            self().asFile().name().compareWithoutCase(spec.filename) == 0)
                        {
                            removeGameTags(meta);
                            meta.appendUniqueWord(VAR_TAGS, "doom2");
                            meta.appendUniqueWord(VAR_TAGS, "masterlevels");
                            break;
                        }
                    }
                }
            }
        }

        determineGameTags(meta);

        LOG_RES_VERBOSE("Identified \"%s\" %s %s score: %i")
                << packageId
                << meta.gets(VAR_VERSION)
                << ::internal::formatDescriptions[format]
                   << meta.geti(VAR_BUNDLE_SCORE); // matched.bestScore;

        return meta;
    }

    void checkAuxiliaryNotes(Record &meta)
    {
        if (format == Pwad || format == Iwad)
        {
            String const dataFilePath = self().asFile().path();
            if (File const *wadTxt = FS::tryLocate<File const>(
                        dataFilePath.fileNameAndPathWithoutExtension() + ".txt"))
            {
                Block txt;
                *wadTxt >> txt;
                meta.set(VAR_NOTES, _E(m) + String::fromCP437(txt));
            }
        }
    }

    /**
     * Determines if the data bundle is intended to be automatically loaded by Doomsday
     * according to the v1.x autoload rules.
     */
    bool isAutoLoaded() const
    {
        Path const path(self().asFile().path());

        //qDebug() << "checking" << path.toString();

        if (path.segmentCount() >= 3)
        {
            String const parent      = path.reverseSegment(1).toString().toLower();
            String const grandParent = path.reverseSegment(2).toString().toLower();

            if (parent.fileNameExtension() == ".pk3" ||
                parent.fileNameExtension() == ".zip" /*||
                parent.fileNameExtension() == ".box"*/)
            {
                // Data files in the root of a PK3/box are all automatically loaded.
                //qDebug() << "-> autoload";
                return true;
            }
            if (//parent.fileNameExtension().isEmpty() &&
                (/*parent == "auto" || */ parent.beginsWith("#") || parent.beginsWith("@")))
            {
                //qDebug() << "-> autoload";
                return true;
            }

//            if (grandParent.fileNameExtension() == ".box")
//            {
//                if (parent == "required")
//                {
//                    return true;
//                }
//                /// @todo What about "Extra"?
//            }
        }

        for (int i = 1; i < path.segmentCount() - 3; ++i)
        {
            if ((path.segment(i) == "Defs" || path.segment(i) == "Data") &&
                (path.segment(i + 1) == "jDoom" || path.segment(i + 1) == "jHeretic" || path.segment(i + 1) == "jHexen") &&
                 path.segment(i + 2) == "Auto")
            {
                //qDebug() << "-> autoload";
                return true;
            }
        }

        //qDebug() << "NOT AUTOLOADED";
        return false;
    }

    /**
     * Reads a Snowberry-style Info file and extracts the relevant parts into the
     * Doomsday 2 package metadata record.
     *
     * @param infoFile  Snowberry Info file.
     * @param meta      Package metadata.
     */
    void parseSnowberryInfo(File const &infoFile, Record &meta)
    {
        Info info;
        String parseErrorMsg;
        try
        {
            info.parse(infoFile);
        }
        catch (Error const &er)
        {
            parseErrorMsg = er.asText();
        }
        auto const &rootBlock = info.root();

        // Tag it as a Snowberry package.
        meta.appendUniqueWord(VAR_TAGS, "legacy");

        if (rootBlock.contains("name"))
        {
            meta.set(Package::VAR_TITLE, rootBlock.keyValue("name"));
        }

        String component = rootBlock.keyValue("component");
        if (!component.isEmpty())
        {
            if (!component.compareWithoutCase("game-jdoom"))
            {
                meta.appendUniqueWord(VAR_TAGS, "doom");
                meta.appendUniqueWord(VAR_TAGS, "doom2");
            }
            else if (!component.compareWithoutCase("game-jheretic"))
            {
                meta.appendUniqueWord(VAR_TAGS, "heretic");
            }
            else if (!component.compareWithoutCase("game-jhexen"))
            {
                meta.appendUniqueWord(VAR_TAGS, "hexen");
            }
        }

        String category = rootBlock.keyValue("category");
        if (!category.isEmpty())
        {
            category.replace("/", "");
            category.replace(" ", "");
            category.replace("gamedata", "data"); // "gamedata" is a special tag
            category.replace("core", ""); // "core" is special tag
            category = category.trimmed();
            if (!category.isEmpty())
            {
                meta.appendUniqueWord(VAR_TAGS, category);
                meta.set(VAR_CATEGORY, category);
            }
        }

        if (Info::BlockElement const *english = rootBlock.findAs<Info::BlockElement>("english"))
        {
            if (english->blockType() == "language")
            {
                // Doomsday must understand the version number.
                Version const sbVer(english->keyValue(VAR_VERSION));
                if (sbVer.isValid())
                {
                    meta.set(VAR_VERSION, sbVer.fullNumber());
                }
                meta.set(VAR_AUTHOR,  english->keyValue("author"));
                meta.set(VAR_LICENSE, english->keyValue("license"));
                meta.set("contact", english->keyValue("contact"));

                String notes = english->keyValue("readme").text.strip();
                if (!notes.isEmpty())
                {
                    notes.replace(QRegExp("\\s+"), " "); // normalize whitespace
                    notes.remove('\r'); // begone foul MS-DOS
                    meta.set(VAR_NOTES, notes);
                }
            }
        }

        if (parseErrorMsg)
        {
            meta.appendUniqueWord(VAR_TAGS, "error");
            meta.set(VAR_NOTES, QObject::tr("There is an error in the metadata of this package: %1")
                .arg(parseErrorMsg) + "\n\n" + meta.gets(VAR_NOTES, ""));
        }
    }

    /**
     * Parses the "notes" entry for metadata in commonly used templates for WAD readmes.
     */
    void parseNotesForMetadata(Record &meta)
    {
        static QRegularExpression const reTitle
                ("^[\\s\x1Bm]*Title\\s*:?\\s*(.*)", QRegularExpression::CaseInsensitiveOption);
        static QRegularExpression const reVersion
                ("^\\s*Version\\s*:\\s*(.*)", QRegularExpression::CaseInsensitiveOption);
        static QRegularExpression const reReleaseDate
                ("^\\s*Release( date)?\\s*:\\s*(.*)", QRegularExpression::CaseInsensitiveOption);
        static QRegularExpression const reAuthor
                ("^\\s*Author(s)?\\s*:?\\s*(.*)", QRegularExpression::CaseInsensitiveOption);
        static QRegularExpression const reContact
                ("^\\s*Email address\\s*:?\\s*(.*)", QRegularExpression::CaseInsensitiveOption);

        bool foundVersion = false;
        bool foundTitle   = false;

        foreach (String line, meta.gets(VAR_NOTES, "").split('\n'))
        {
            if (!foundTitle)
            {
                auto match = reTitle.match(line);
                if (match.hasMatch())
                {
                    meta.set(VAR_TITLE, match.captured(1).trimmed());
                    foundTitle = true;
                    continue;
                }
            }

            if (!foundVersion)
            {
                auto match = reReleaseDate.match(line);
                if (match.hasMatch())
                {
                    Date const releaseDate = Date::fromText(match.captured(2).trimmed());
                    if (releaseDate.isValid())
                    {
                        meta.set(VAR_VERSION, QString("%1.%2.%3")
                                 .arg(releaseDate.year())
                                 .arg(releaseDate.month())
                                 .arg(releaseDate.dayOfMonth()));
                    }
                    continue;
                }
            }

            auto match = reVersion.match(line);
            if (match.hasMatch())
            {
                Version parsed(match.captured(1).trimmed());
                if (parsed.isValid())
                {
                    meta.set(VAR_VERSION, parsed.fullNumber());
                    foundVersion = true;
                }
                continue;
            }

            match = reAuthor.match(line);
            if (match.hasMatch())
            {
                meta.set(VAR_AUTHOR, match.captured(2).trimmed());
                continue;
            }

            match = reContact.match(line);
            if (match.hasMatch())
            {
                meta.set("contact", match.captured(1).trimmed());
                continue;
            }
        }
    }

    bool identifyMostLikelyGame(String const &text, String &identifiedTag)
    {
        if (text.isEmpty()) return false;

        identifiedTag.clear();

        // Look for terms that refer to specific games.
        static QList<std::pair<String, StringList>> terms;
        if (terms.isEmpty())
        {
            terms << std::make_pair(String("doom2"),   StringList({ "\\b(doom2|doom 2|DoomII|Doom II|final\\s*doom|plutonia|tnt)\\b" }));
            terms << std::make_pair(String("doom"),    StringList({ "^doom$|\\bdoom[^ s2][^2d]\\b|\\bultimate\\s*doom\\b|\\budoom\\b" }));
            terms << std::make_pair(String("heretic"), StringList({ "\\b(jheretic|heretic)\\b", "\\b(d'sparil|serpent rider)\\b" }));
            terms << std::make_pair(String("hexen"),   StringList({ "\\b(jhexen|hexen)\\b", "\\b(korax|mage|warrior|cleric)\\b" })); 
        }
        QHash<String, int> scores;
        for (auto i : terms) //= terms.constBegin(); i != terms.constEnd(); ++i)
        {
            for (String const &term : i.second)
            {
                QRegularExpression re(term, QRegularExpression::CaseInsensitiveOption);
                auto match = re.match(text);
                if (match.hasMatch())
                {
                    //qDebug() << "match:" << term << "in" << match.captured() << "scoring for:" << i.first;
                    //scores[i.key()]++;
                    identifiedTag = i.first;
                    return true;
                }
            }
        }
        //if (scores.isEmpty())
        {
            return false;
        }
        /*QList<std::pair<int, String>> sorted;
        for (auto i = scores.constBegin(); i != scores.constEnd(); ++i)
        {
            sorted.append(std::make_pair(i.value(), i.key()));
        }
        qSort(sorted.begin(), sorted.end(), []
              (std::pair<int, String> const &a, std::pair<int, String> const &b) {
            return a.first > b.first;
        });
        //qDebug() << text << sorted;
        identifiedTag = sorted.first().second;
        return true;*/
    }

    static bool containsAnyGameTag(Record const &meta)
    {
        return countGameTags(meta) > 0;
    }

    static bool containsAmbiguousGameTags(Record const &meta)
    {
        return countGameTags(meta) != 1;
    }

    static int countGameTags(Record const &meta)
    {
        int count = 0;
        for (auto const &tag : gameTags())
        {
            if (QRegExp(QString("\\b%1\\b").arg(tag), Qt::CaseInsensitive)
                    .indexIn(meta.gets(VAR_TAGS)) >= 0)
            {
                // Already has at least one game tag.
                count++;
            }
        }
        return count;
    }

    static void removeGameTags(Record &meta)
    {
        String newTags = meta.gets(VAR_TAGS);
        newTags.remove(QRegularExpression(anyGameTagPattern()));
//        foreach (QString tag, Package::tags(meta.gets(VAR_TAGS)))
//        {
//            if (!gameTags().contains(tag))
//            {
//                if (!newTags.isEmpty()) newTags += QStringLiteral(" ");
//                newTags += tag;
//            }
//        }
        meta.set(VAR_TAGS, newTags);
    }

    /**
     * Automatically guesses some appropriate game tags for the bundle.
     * @param meta  Package metadata.
     */
    void determineGameTags(Record &meta)
    {
        //qDebug() << "Determining:" << meta.gets(VAR_TITLE) << meta.gets(VAR_TAGS);

        if (!containsAmbiguousGameTags(meta))
        {
            // Already has exactly one game tag.
            //qDebug() << meta.gets(VAR_TITLE) << "- unambiguous";
            return;
        }

        String const oldTags = meta.gets(VAR_TAGS);

        String tag;
        if (identifyMostLikelyGame(meta.gets(VAR_TITLE), tag))
        {
            //qDebug() << meta.gets(VAR_TITLE)<< "- from title:" << tag;
            removeGameTags(meta);
            meta.appendUniqueWord(VAR_TAGS, tag);
        }
        else if (identifyMostLikelyGame(meta.gets("ID"), tag))
        {
            //qDebug() << meta.gets(VAR_TITLE) << "- from package ID:" << tag;
            removeGameTags(meta);
            meta.appendUniqueWord(VAR_TAGS, tag);
        }
        else if (identifyMostLikelyGame(meta.gets(VAR_NOTES, ""), tag))
        {
            //qDebug() << meta.gets(VAR_TITLE)<< "- from notes:" << tag;
            removeGameTags(meta);
            meta.appendUniqueWord(VAR_TAGS, tag);
        }

        if (!containsAnyGameTag(meta))
        {
            // As a fallback, look at the path to estimate which game it is
            // compatible with.
            Path const path(self().asFile().path());
            for (int i = 0; i < path.segmentCount(); ++i)
            {
                if (identifyMostLikelyGame(path.segment(i), tag))
                {
                    //qDebug() << meta.gets(VAR_TITLE)<< "- from path:" << tag;
                    meta.appendUniqueWord(VAR_TAGS, tag);
                }
            }
        }

        if (!containsAnyGameTag(meta))
        {
            //qDebug() << meta.gets(VAR_TITLE)<< "- falling back to:" << oldTags;
            // Failed to figure out anything, so fall back to the old tags.
            meta.set(VAR_TAGS, oldTags);
        }
    }

    struct PathAndVersion {
        String path;
        String version;
        PathAndVersion(String const &path = String(), String const &version = String())
            : path(path), version(version) {}
        operator bool() { return !path.isEmpty(); }
    };

    PathAndVersion chooseUniqueLinkPathAndVersion(File const &dataFile,
                                                  String const &packageId,
                                                  Version const &packageVersion,
                                                  dint bundleScore)
    {
        for (int attempt = 0; attempt < 3; ++attempt)
        {
            String linkPath = packageId;
            String version = (packageVersion.isValid()? packageVersion.fullNumber() : "");

            // Try a few different ways to generate a locally unique version number.
            switch (attempt)
            {
            case 0: // unmodified
                break;

            case 1: // parse version from parent folder
            //case 2: // parent folder as version label
                if (dataFile.path().fileNamePath() != "/local/wads")
                {
                    Path const filePath(dataFile.path());
                    if (filePath.segmentCount() >= 2)
                    {
                        auto const &parentName = filePath.reverseSegment(1)
                                .toString().fileNameWithoutExtension();
                        //if (attempt == 1)
                        {
                            Version parsed("");
                            stripVersion(parentName, &parsed);
                            if (parsed.isValid())
                            {
                                version = parsed.fullNumber();
                            }
                        }
                        /*else
                        {
                            version = "1.0-" + filePath.reverseSegment(1)
                                    .toString().fileNameWithoutExtension().toLower();
                        }*/
                    }
                }
                break;

            case 2: // version from status
                // Larger versions are preferred when multiple versions are available,
                // so use the major version 0 to avoid always preferring these date-based
                // versions.
                version = dataFile.status().modifiedAt.asDateTime().toString("0.yyyy.MMdd.hhmm");
                break;
            }

            if (!version.isEmpty())
            {
                DENG2_ASSERT(Version(version).isValid());
                linkPath += QString("_%1.pack").arg(version);
            }
            else
            {
                linkPath += QStringLiteral(".pack");
            }

            //qDebug() << "checking" << linkPath;

            // Each link must have a unique name.
            if (!bundleFolder().has(linkPath))
            {
                return PathAndVersion(linkPath, version);
            }
            else
            {
                auto const &file = bundleFolder().locate<File const>(linkPath);

                if (LinkFile const *linkFile = maybeAs<LinkFile>(file))
                {
                    if (linkFile->isBroken())
                    {
                        // This can be replaced.
                        bundleFolder().destroyFile(linkPath);
                        return PathAndVersion(linkPath, version);
                    }
                }

                // This could still be a better scored match.
                if (bundleScore > file.objectNamespace().geti("package.bundleScore"))
                {
                    // Forget about the previous link.
                    bundleFolder().destroyFile(linkPath);
                    return PathAndVersion(linkPath, version);
                }
            }
        }

        // Unique path & version not available. This version of the package is probably
        // already available.
        LOG_RES_XVERBOSE("Failed to make a unique link for %s (%s %s score:%i)",
                         dataFile.description() << packageId
                         << packageVersion.fullNumber() << bundleScore);
        return PathAndVersion();
    }

    String guessCompatibleGame() const
    {
        if (!pkgLink) return String();

        QSet<QString> tags;
        foreach (QString tag, Package::tags(*pkgLink)) tags.insert(tag);

        // Check for tags matching game IDs.
        /*
        {
            String matchingGameId;
            if (DoomsdayApp::games().forAll([&tags, &matchingGameId] (Game &game) {
                    QRegularExpression re(QString("\\b%1\\b").arg(game.id()));
                    if (re.match(text).hasMatch())
                    {
                        matchinGameId = game.id();
                        return LoopAbort;
                    }
                    return LoopContinue;
                }))
            {
                return matchingGameId;
            }
        }*/

        res::LumpDirectory::MapType const mapType = lumpDir? lumpDir->mapType()
                                                           : res::LumpDirectory::None;

        if (tags.contains("doom") || tags.contains("doom2"))
        {
            if (mapType == res::LumpDirectory::MAPxx || !tags.contains("doom"))
            {
                return "doom2";
            }
            else
            {
                if (Games::get()["doom1-ultimate"].isPlayable())
                {
                    return "doom1-ultimate";
                }
                return "doom1";
            }
        }
        if (tags.contains("hexen"))
        {
            return "hexen";
        }
        if (tags.contains("heretic"))
        {
            if (mapType == res::LumpDirectory::MAPxx)
            {
                return "hexen";
            }
            if (Games::get()["heretic-ext"].isPlayable())
            {
                return "heretic-ext";
            }
            return "heretic";
        }
        // We couldn't figure it out.
        return String();
    }
};

DataBundle::DataBundle(Format format, File &source)
    : d(new Impl(this, format))
{
    d->source.reset(&source);
}

DataBundle::~DataBundle()
{}

DataBundle::Format DataBundle::format() const
{
    return d->format;
}

String DataBundle::formatAsText() const
{
    return ::internal::formatDescriptions[d->format];
}

String DataBundle::description() const
{
    if (!d->source)
    {
        return "invalid data bundle";
    }
    return QString("%1 %2")
            .arg(::internal::formatDescriptions[d->format])
            .arg(d->source->description());
}

File &DataBundle::asFile()
{
    return *dynamic_cast<File *>(this);
}

File const &DataBundle::asFile() const
{
    return *dynamic_cast<File const *>(this);
}

File const &DataBundle::sourceFile() const
{
    return *asFile().source();
}

String DataBundle::rootPath() const
{
    return asFile().path().fileNamePath();
}

String DataBundle::packageId() const
{
    if (!d->packageId)
    {
        identifyPackages();
    }
    return d->packageId;
}

String DataBundle::versionedPackageId() const
{
    if (!d->packageId)
    {
        identifyPackages();
    }
    return d->versionedPackageId;
}

IByteArray::Size DataBundle::size() const
{
    if (d->source)
    {
        return d->source->size();
    }
    return 0;
}

void DataBundle::get(Offset at, Byte *values, Size count) const
{
    if (!d->source)
    {
        throw File::InputError("DataBundle::get", "Source file has been destroyed");
    }
    d->source->as<ByteArrayFile>().get(at, values, count);
}

void DataBundle::set(Offset, Byte const *, Size)
{
    throw File::OutputError("DataBundle::set", "Classic data formats are read-only");
}

Record &DataBundle::objectNamespace()
{
    DENG2_ASSERT(dynamic_cast<File *>(this) != nullptr);
    return asFile().objectNamespace().subrecord(QStringLiteral("package"));
}

Record const &DataBundle::objectNamespace() const
{
    DENG2_ASSERT(dynamic_cast<File const *>(this) != nullptr);
    return asFile().objectNamespace().subrecord(QStringLiteral("package"));
}

DataBundle::Format DataBundle::packageBundleFormat(String const &packageId) // static
{
    if (auto const *bundle = bundleForPackage(packageId))
    {
        Guard g(bundle->d);
        return bundle->format();
    }
    return Unknown;
}

DataBundle const *DataBundle::bundleForPackage(String const &packageId) // static
{
    if (File const *file = PackageLoader::get().select(packageId))
    {
        if (auto const *bundle = maybeAs<DataBundle>(file->target()))
        {
            return bundle;
        }
    }
    return nullptr;
}

DataBundle const *DataBundle::tryLocateDataFile(Package const &package, String const &dataFilePath)
{
    if (DataBundle const *bundle = package.root().tryLocate<DataBundle const>(dataFilePath))
    {
        return bundle;
    }
    // The package may itself be a link to a data bundle.
    if (DataBundle const *bundle = maybeAs<DataBundle>(package.sourceFile().target()))
    {
        return bundle;
    }
    return nullptr;
}

void DataBundle::setFormat(Format format)
{
    d->format = format;
}

bool DataBundle::identifyPackages() const
{
    LOG_AS("DataBundle");
    try
    {
        return d->identify();
    }
    catch (Error const &er)
    {
        LOG_RES_WARNING("Failed to identify %s: %s") << description() << er.asText();
    }
    return false;
}

bool DataBundle::isLinkedAsPackage() const
{
    return bool(d->pkgLink);
}

Record &DataBundle::packageMetadata()
{
    if (!isLinkedAsPackage())
    {
        throw LinkError("DataBundle::packageMetadata", "Data bundle " +
                        description() + " has not been identified and linked as a package");
    }
    return d->pkgLink->objectNamespace().subrecord(Package::VAR_PACKAGE);
}

Record const &DataBundle::packageMetadata() const
{
    return const_cast<DataBundle *>(this)->packageMetadata();
}

bool DataBundle::isNested() const
{
    return containerBundle() != nullptr || !containerPackageId().isEmpty();
}

DataBundle *DataBundle::containerBundle() const
{
    auto const *file = dynamic_cast<File const *>(this);
    DENG2_ASSERT(file != nullptr);

    for (Folder *folder = file->parent(); folder; folder = folder->parent())
    {
        if (auto *data = maybeAs<DataFolder>(folder))
        {
            return data;
        }
    }
    return nullptr;
}

String DataBundle::containerPackageId() const
{
    auto const *file = dynamic_cast<File const *>(this);
    DENG2_ASSERT(file != nullptr);

    return Package::identifierForContainerOfFile(*file);
}

res::LumpDirectory const *DataBundle::lumpDirectory() const
{
    return d->lumpDir.get();
}

String DataBundle::guessCompatibleGame() const
{
    return d->guessCompatibleGame();
}

File *DataBundle::Interpreter::interpretFile(File *sourceData) const
{
    // Broken links cannot be interpreted.
    if (LinkFile *link = maybeAs<LinkFile>(sourceData))
    {
        if (link->isBroken()) return nullptr;
    }

    // Naive check using the file extension.
    static struct { String str; Format format; } formats[] = {
        { ".pk3.zip", Pk3 },
        { ".pk3",     Pk3 },
        { ".wad",     Wad /* type (I or P) checked later */ },
        { ".lmp",     Lump },
        { ".ded",     Ded },
        { ".deh",     Dehacked },
        { ".box",     Collection },
    };
    //String const ext = sourceData->extension();
    for (auto const &fmt : formats)
    {
        if (sourceData->name().endsWith(fmt.str, String::CaseInsensitive))
        {
            LOG_RES_XVERBOSE("Interpreted %s as %s",
                             sourceData->description() <<
                             ::internal::formatDescriptions[fmt.format]);

            switch (fmt.format)
            {
            case Pk3:
            case Collection:
                return new DataFolder(fmt.format, *sourceData);

            default:
                return new DataFile(fmt.format, *sourceData);
            }
        }
    }
    // Was not interpreted.
    return nullptr;
}

QList<DataBundle const *> DataBundle::loadedBundles() // static
{
    QList<DataBundle const *> loaded;

    // Check all the loaded packages to see which ones are data bundles.
    for (auto *f : PackageLoader::get().loadedPackagesAsFilesInPackageOrder())
    {
        if (DataBundle const *bundle = maybeAs<DataBundle>(f))
        {
            // Non-collection data files are loaded as-is.
            loaded << bundle;
        }
        else
        {
            // Packages may declare a list of data files to load.
            Package const *pkg = PackageLoader::get().tryFindLoaded(*f);
            DENG2_ASSERT(pkg);

            auto const &meta = Package::metadata(*f);
            if (meta.has(VAR_DATA_FILES))
            {
                foreach (Value const *v, meta.geta(VAR_DATA_FILES).elements())
                {
                    String const dataFilePath = v->asText();

                    // Look up the data bundle file.
                    if (DataBundle const *bundle = tryLocateDataFile(*pkg, dataFilePath))
                    {
                        // Identify it now (if not already identified). Note that data
                        // files inside packages usually aren't identified during
                        // startup.
                        bundle->identifyPackages();
                        if (bundle->isLinkedAsPackage())
                        {
                            loaded << bundle;
                        }
                        else
                        {
                            LOG_RES_WARNING("Cannot identify %s") << bundle->asFile().description();
                        }
                    }
                    else
                    {
                        LOG_RES_WARNING("Cannot load \"%s\" from %s") << dataFilePath << f->description();
                    }
                }
            }
        }
    }

    return loaded;
}

QList<DataBundle const *> DataBundle::findAllNative(String const &fileNameOrPartialNativePath)
{
    NativePath const searchPath = NativePath(fileNameOrPartialNativePath).expand();

    FS::FoundFiles found;
    FS::get().findAllOfTypes(StringList({ DENG2_TYPE_NAME(DataFile),
                                          DENG2_TYPE_NAME(DataFolder) }),
                             searchPath.fileName().toLower(), found);
    QList<DataBundle const *> bundles;
    for (auto const *f : found)
    {
        DENG2_ASSERT(dynamic_cast<DataBundle const *>(f));
        bundles << dynamic_cast<DataBundle const *>(f);
    }

    // Omit the ones that don't match the given native path.
    if (!searchPath.fileNamePath().isEmpty())
    {
        bundles = de::filter(bundles, [&searchPath] (DataBundle const *b)
        {
            NativePath const bundlePath = b->asFile().correspondingNativePath().fileNamePath();
            if (bundlePath.isEmpty()) return false;
            //qDebug() << "bundle:" << path.asText() << "searchTerm:" << searchPath.fileNamePath();
            if (bundlePath.toString().endsWith(searchPath.fileNamePath(),
                                               String::CaseInsensitive))
            {
                return true;
            }
            return false;
        });
    }

    return bundles;
}

StringList DataBundle::gameTags()
{
    static StringList const gameTags({ "doom", "doom2", "heretic", "hexen" });
    return gameTags;
}

String DataBundle::anyGameTagPattern()
{
    return String("\\b(%1)\\b").arg(String::join(gameTags(), "|"));
}

String DataBundle::cleanIdentifier(String const &text)
{
    // Periods and underscores have special meaning in packages IDs.
    // Whitespace is used as separator in package ID lists (see PackageLoader).
    // Info syntax has ambiguous quote/double-quote escaping in strings, so
    // we'll also get rid of single quotes. (For example, Info converts a string
    // containing ['"] to ['''].)
    String cleaned = text.toLower();
    cleaned.replace(QRegExp("[._'\\s]"), "-");
    return cleaned;
}

String DataBundle::stripVersion(String const &text, Version *version)
{
    QRegExp re(".*([-_. ]v?([0-9._-]+))$");
    if (re.exactMatch(text))
    {
        if (version)
        {
            String str = re.cap(2);
            str.replace("_", ".");
            version->parseVersionString(str);
        }
        return text.mid(0, text.size() - re.cap(1).size());
    }
    return text;
}

String DataBundle::stripRedundantParts(String const &id)
{
    DotPath const path(id);
    String stripped = path.segment(0);
    for (int i = 1; i < path.segmentCount(); ++i)
    {
        String seg = path.segment(i);
        for (int k = 1; k <= i; ++k) // Check all previous segments.
        {
            if (seg.startsWith(path.segment(i - k) + "-"))
            {
                seg = seg.mid(path.segment(i - k).size() + 1);
                break;
            }
        }
        stripped = stripped.concatenateMember(seg);
    }
    return stripped;
}

de::String DataBundle::versionFromTimestamp(const Time &timestamp)
{
    return timestamp.asDateTime().toString(QStringLiteral("0.yyyy.MMdd.hhmm"));
}

void DataBundle::checkAuxiliaryNotes(Record &packageMetadata)
{
    d->checkAuxiliaryNotes(packageMetadata);
}
