diff --git a/codegen/lang/CMakeLists.txt b/codegen/lang/CMakeLists.txt index f2a6a83..e5097fa 100644 --- a/codegen/lang/CMakeLists.txt +++ b/codegen/lang/CMakeLists.txt @@ -14,6 +14,8 @@ PRIVATE codegen/lang/parsed_file.h codegen/lang/processor.cpp codegen/lang/processor.h + codegen/lang/subsets.cpp + codegen/lang/subsets.h ) target_include_directories(codegen_lang diff --git a/codegen/lang/generator.cpp b/codegen/lang/generator.cpp index c12de72..07e5b60 100644 --- a/codegen/lang/generator.cpp +++ b/codegen/lang/generator.cpp @@ -6,42 +6,195 @@ // #include "codegen/lang/generator.h" +#include +#include #include #include #include +#include #include +#include #include #include namespace codegen { namespace lang { +namespace { + +constexpr int kErrorTooManyKeys = 841; +constexpr int kErrorTooManyTags = 842; + +constexpr auto kIndexLimit = std::numeric_limits::max(); + +struct Slot { + int start = 0; + int size = 0; +}; + +} // namespace Generator::Generator(const LangPack &langpack, const QString &destBasePath, const common::ProjectInfo &project) : langpack_(langpack) , basePath_(destBasePath) , baseName_(QFileInfo(basePath_).baseName()) , project_(project) { + allocateIndices(); + saveTagOrder(); + collectDeclarations(); +} + +void Generator::saveTagOrder() { + auto file = QFile(basePath_ + ".tags"); + if (file.open(QIODevice::WriteOnly)) { + auto stream = QTextStream(&file); + for (const auto &tag : langpack_.tags) { + stream << tag.tag << '\n'; + } + } +} + +void Generator::allocateIndices() { + const auto path = basePath_ + ".indices"; + auto known = std::map(); + auto next = 0; + + auto reading = QFile(path); + if (reading.open(QIODevice::ReadOnly)) { + auto stream = QTextStream(&reading); + while (!stream.atEnd()) { + const auto parts = stream.readLine().split('\t'); + if (parts.size() != 3) { + continue; + } + const auto start = parts[1].toInt(); + const auto size = parts[2].toInt(); + if (size <= 0 || start < 0) { + continue; + } + known.emplace(parts[0], Slot{ start, size }); + next = std::max(next, start + size); + } + reading.close(); + } + + indices_.assign(langpack_.entries.size(), -1); + for (auto i = 0, count = int(langpack_.entries.size()); i != count; ++i) { + const auto &entry = langpack_.entries[i]; + const auto isPlural = !entry.keyBase.isEmpty(); + if (isPlural && entry.key != ComputePluralKey(entry.keyBase, 0)) { + continue; // Taken by the first part of the plural group. + } + const auto name = isPlural ? entry.keyBase : entry.key; + const auto size = isPlural ? kPluralPartCount : 1; + auto j = known.find(name); + if (j == known.end() || j->second.size != size) { + j = known.insert_or_assign(name, Slot{ next, size }).first; + next += size; + } + for (auto shift = 0; shift != size; ++shift) { + indices_[i + shift] = j->second.start + shift; + } + } + keysCount_ = next; + if (keysCount_ > kIndexLimit) { + common::logError(kErrorTooManyKeys, path) + << "the key table needs " << keysCount_ + << " slots, the limit is " << kIndexLimit + << ". Removed keys keep their slot so that re-adding one is free; " + << "delete this file to renumber the keys from scratch."; + failed_ = true; + return; + } + + auto writing = QFile(path); + if (writing.open(QIODevice::WriteOnly)) { + auto stream = QTextStream(&writing); + for (const auto &[name, slot] : known) { + stream << name << '\t' << slot.start << '\t' << slot.size << '\n'; + } + } +} + +void Generator::collectDeclarations() { + auto byIndex = std::map(); + for (auto i = 0, count = int(langpack_.entries.size()); i != count; ++i) { + const auto &entry = langpack_.entries[i]; + const auto isPlural = !entry.keyBase.isEmpty(); + const auto &key = entry.key; + if (isPlural && key != ComputePluralKey(entry.keyBase, 0)) { + continue; + } + auto tags = QStringList(); + for (auto &tagData : entry.tags) { + tags.push_back("lngtag_" + tagData.tag); + } + byIndex.emplace(indices_[i], "inline constexpr phrase<" + + tags.join(", ") + + "> " + + (isPlural ? entry.keyBase : key) + + "{ ushort(" + + QString::number(indices_[i]) + + ") };"); + } + declarations_.reserve(byIndex.size()); + for (auto &[index, declaration] : byIndex) { + declarations_.push_back(declaration); + } } bool Generator::writeHeader() { + if (failed_) { + return false; + } else if (int(langpack_.tags.size()) > kIndexLimit) { + common::logError(kErrorTooManyTags, basePath_ + ".tags") + << "there are " << langpack_.tags.size() + << " tags, the limit is " << kIndexLimit + << ". Delete this file to drop the tags that are not used anymore."; + return false; + } header_ = std::make_unique(basePath_ + ".h", project_); header_->include("lang/lang_tag.h").include("lang/lang_values.h").newline(); - writeHeaderForwardDeclarations(); writeHeaderTagTypes(); writeHeaderInterface(); writeHeaderReactiveInterface(); + writeHeaderKeysInclude(); return header_->finalize(); } -void Generator::writeHeaderForwardDeclarations() { - header_->pushNamespace("Lang").stream() << "\ +void Generator::writeHeaderKeysInclude() { + header_->stream() << "\ +#ifdef LANG_KEYS_SUBSET\n\ +#include LANG_KEYS_SUBSET\n\ +#else // LANG_KEYS_SUBSET\n\ +#include \"" << baseName_ << "_keys.h\"\n\ +#endif // LANG_KEYS_SUBSET\n"; +} + +bool Generator::writeCounts() { + auto file = common::CppFile(basePath_ + "_counts.h", project_); + file.pushNamespace("Lang").stream() << "\ \n\ inline constexpr auto kTagsCount = ushort(" << langpack_.tags.size() << ");\n\ -inline constexpr auto kKeysCount = ushort(" << langpack_.entries.size() << ");\n\ +inline constexpr auto kKeysCount = ushort(" << keysCount_ << ");\n\ \n"; - header_->popNamespace().newline(); + file.popNamespace(); + return file.finalize(); +} + +bool Generator::writeAllKeys() { + auto file = common::CppFile(basePath_ + "_keys.h", project_); + file.include(baseName_ + ".h").newline(); + file.pushNamespace("tr").newline(); + for (const auto &[combination, code] : specializations_) { + file.stream() << code; + } + for (const auto &declaration : declarations_) { + file.stream() << declaration << "\n"; + } + file.newline().popNamespace(); + return file.finalize(); } void Generator::writeHeaderTagTypes() { @@ -98,7 +251,6 @@ void Generator::writeHeaderReactiveInterface() { header_->pushNamespace("tr"); writeHeaderProducersInterface(); - writeHeaderProducersInstances(); header_->popNamespace().newline(); } @@ -132,7 +284,7 @@ using S = std::decay_t()(QString()))>;\n\ template \n\ struct phrase;\n\ \n"; - std::set producersDeclared; + auto specializations = std::map(); for (auto &entry : langpack_.entries) { const auto isPlural = !entry.keyBase.isEmpty(); auto tags = QStringList(); @@ -160,58 +312,62 @@ struct phrase;\n\ } producerArgs.push_back("P p = P()"); currentArgs.push_back("P p = P()"); - if (!producersDeclared.emplace(tags.join(',')).second) { + const auto combination = tags.join(','); + if (specializations.find(combination) != specializations.end()) { continue; } - header_->stream() << "\ + specializations.emplace(combination, "\ template <>\n\ -struct phrase<" << tags.join(", ") << "> {\n\ +struct phrase<" + tags.join(", ") + "> {\n\ template \n\ - rpl::producer> operator()(" << producerArgs.join(", ") << ") const {\n\ - return ::Lang::details::Producer<" << tags.join(", ") << ">::Combine(" << values.join(", ") << ");\n\ + rpl::producer> operator()(" + producerArgs.join(", ") + ") const {\n\ + return ::Lang::details::Producer<" + tags.join(", ") + ">::Combine(" + values.join(", ") + ");\n\ }\n\ \n\ template \n\ - S

operator()(now_t, " << currentArgs.join(", ") << ") const {\n\ - return ::Lang::details::Producer<" << tags.join(", ") << ">::Current(" << values.join(", ") << ");\n\ + S

operator()(now_t, " + currentArgs.join(", ") + ") const {\n\ + return ::Lang::details::Producer<" + tags.join(", ") + ">::Current(" + values.join(", ") + ");\n\ }\n\ \n\ ushort base;\n\ };\n\ -\n"; +\n"); } -} - -void Generator::writeHeaderProducersInstances() { - auto index = 0; - for (auto &entry : langpack_.entries) { - const auto isPlural = !entry.keyBase.isEmpty(); - const auto &key = entry.key; - auto tags = QStringList(); - for (auto &tagData : entry.tags) { - const auto &tag = tagData.tag; - tags.push_back("lngtag_" + tag); - } - if (!isPlural || key == ComputePluralKey(entry.keyBase, 0)) { - header_->stream() << "\ -inline constexpr phrase<" << tags.join(", ") << "> " << (isPlural ? entry.keyBase : key) << "{ ushort(" << index << ") };\n"; - } - ++index; + const auto plain = specializations.find(QString()); + if (plain != specializations.end()) { + header_->stream() << plain->second; + specializations.erase(plain); } - header_->newline(); + specializations_ = std::move(specializations); } bool Generator::writeSource() { source_ = std::make_unique(basePath_ + ".cpp", project_); - source_->include("lang/lang_keys.h").pushNamespace("Lang").pushNamespace(); + source_->include("lang/lang_keys.h") + .include(baseName_ + "_counts.h") + .pushNamespace("Lang") + .pushNamespace(); + + source_->stream() << "\ +static_assert(sizeof(QChar) == sizeof(char16_t));\n\ +static_assert(alignof(QChar) == alignof(char16_t));\n\ +\n"; + + auto byIndex = std::vector(keysCount_, nullptr); + for (auto i = 0, count = int(langpack_.entries.size()); i != count; ++i) { + byIndex[indices_[i]] = &langpack_.entries[i]; + } source_->stream() << "\ -QChar DefaultData[] = {"; +const char16_t DefaultData[] = {"; auto count = 0; auto fulllength = 0; - for (auto &entry : langpack_.entries) { - for (auto ch : entry.value) { + for (const auto entry : byIndex) { + if (!entry) { + continue; + } + for (auto ch : entry->value) { if (fulllength > 0) source_->stream() << ","; if (!count++) { source_->stream() << "\n"; @@ -221,7 +377,7 @@ QChar DefaultData[] = {"; } source_->stream() << " "; } - source_->stream() << "QChar(0x" << QString::number(ch.unicode(), 16) << ")"; + source_->stream() << "0x" << QString::number(ch.unicode(), 16); ++fulllength; } } @@ -230,8 +386,9 @@ QChar DefaultData[] = {"; int Offsets[] = {"; count = 0; auto offset = 0; - auto writeOffset = [this, &count, &offset] { - if (offset > 0) source_->stream() << ","; + auto written = 0; + auto writeOffset = [this, &count, &offset, &written] { + if (written++ > 0) source_->stream() << ","; if (!count++) { source_->stream() << "\n"; } else { @@ -242,9 +399,9 @@ int Offsets[] = {"; } source_->stream() << offset; }; - for (auto &entry : langpack_.entries) { + for (const auto entry : byIndex) { writeOffset(); - offset += entry.value.size(); + offset += entry ? entry->value.size() : 0; } writeOffset(); source_->stream() << " };\n"; @@ -270,10 +427,11 @@ ushort GetKeyIndex(QLatin1String key) {\n\ auto size = key.size();\n\ auto data = key.data();\n"; - auto index = 0; auto indices = std::map(); - for (auto &entry : langpack_.entries) { - indices.emplace(getFullKey(entry), QString::number(index++)); + for (auto i = 0, total = int(langpack_.entries.size()); i != total; ++i) { + indices.emplace( + getFullKey(langpack_.entries[i]), + QString::number(indices_[i])); } const auto indexOfKey = [&](const QString &full) { const auto i = indices.find(full); @@ -354,7 +512,9 @@ QString GetOriginalValue(ushort key) {\n\ Expects(key < kKeysCount);\n\ \n\ const auto offset = Offsets[key];\n\ - return QString::fromRawData(DefaultData + offset, Offsets[key + 1] - offset);\n\ + return QString::fromRawData(\n\ + reinterpret_cast(DefaultData + offset),\n\ + Offsets[key + 1] - offset);\n\ }\n\ \n"; diff --git a/codegen/lang/generator.h b/codegen/lang/generator.h index f666d35..ddf2e74 100644 --- a/codegen/lang/generator.h +++ b/codegen/lang/generator.h @@ -25,16 +25,23 @@ class Generator { Generator &operator=(const Generator &other) = delete; bool writeHeader(); + + bool writeCounts(); + bool writeAllKeys(); + bool writeSource(); private: - void writeHeaderForwardDeclarations(); + void allocateIndices(); + void saveTagOrder(); + void collectDeclarations(); + void writeHeaderTagTypes(); void writeHeaderInterface(); void writeHeaderTagValueLookup(); void writeHeaderReactiveInterface(); void writeHeaderProducersInterface(); - void writeHeaderProducersInstances(); + void writeHeaderKeysInclude(); QString getFullKey(const LangPack::Entry &entry); @@ -46,6 +53,14 @@ class Generator { const common::ProjectInfo &project_; std::unique_ptr source_, header_; + std::vector indices_; + int keysCount_ = 0; + bool failed_ = false; + + std::vector declarations_; + + std::map specializations_; + }; } // namespace lang diff --git a/codegen/lang/options.cpp b/codegen/lang/options.cpp index 2237fa3..c788575 100644 --- a/codegen/lang/options.cpp +++ b/codegen/lang/options.cpp @@ -19,6 +19,7 @@ constexpr int kErrorOutputPathExpected = 902; constexpr int kErrorInputPathExpected = 903; constexpr int kErrorSingleInputPathExpected = 904; constexpr int kErrorWorkingPathExpected = 905; +constexpr int kErrorSourcesPathExpected = 906; } // namespace @@ -52,6 +53,21 @@ Options parseOptions() { } else if (arg.startsWith("-w")) { common::logSetWorkingPath(arg.mid(2)); + // Sources path + } else if (arg == "-s") { + if (++i == count) { + logError(kErrorSourcesPathExpected, "Command Line") << "sources path expected after -s"; + return Options(); + } else { + result.sourcesPath = args.at(i); + } + } else if (arg.startsWith("-s")) { + result.sourcesPath = arg.mid(2); + + // Subsets only + } else if (arg == "--subsets-only") { + result.subsetsOnly = true; + // Input path } else { if (result.inputPath.isEmpty()) { diff --git a/codegen/lang/options.h b/codegen/lang/options.h index a0cec21..1ac840f 100644 --- a/codegen/lang/options.h +++ b/codegen/lang/options.h @@ -15,6 +15,10 @@ namespace lang { struct Options { QString outputPath = "."; QString inputPath; + + QString sourcesPath; + + bool subsetsOnly = false; }; // Parsing failed if inputPath is empty in the result. diff --git a/codegen/lang/parsed_file.cpp b/codegen/lang/parsed_file.cpp index 0f01310..b7a91c9 100644 --- a/codegen/lang/parsed_file.cpp +++ b/codegen/lang/parsed_file.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include "codegen/common/basic_tokenized_file.h" #include "codegen/common/logging.h" #include "base/qt/qt_string_view.h" @@ -86,6 +87,7 @@ bool ParsedFile::read() { if (!file_.read()) { return false; } + loadTagOrder(); do { if (auto keyToken = file_.getToken(BasicType::String)) { @@ -116,6 +118,20 @@ bool ParsedFile::read() { return !failed(); } +void ParsedFile::loadTagOrder() { + auto file = QFile(options_.outputPath + "/lang_auto.tags"); + if (!file.open(QIODevice::ReadOnly)) { + return; + } + auto stream = QTextStream(&file); + while (!stream.atEnd()) { + const auto tag = stream.readLine().trimmed(); + if (!tag.isEmpty()) { + result_.tags.push_back({ tag }); + } + } +} + void ParsedFile::fillPluralTags() { auto count = result_.entries.size(); for (auto i = 0; i != count;) { diff --git a/codegen/lang/parsed_file.h b/codegen/lang/parsed_file.h index 416f566..02e51be 100644 --- a/codegen/lang/parsed_file.h +++ b/codegen/lang/parsed_file.h @@ -92,6 +92,8 @@ class ParsedFile { QString extractTagsData(const QString &value, LangPack *to); QString extractTagData(const QString &tag, LangPack *to); + void loadTagOrder(); + void fillPluralTags(); QString filePath_; diff --git a/codegen/lang/processor.cpp b/codegen/lang/processor.cpp index 3c4ced2..5008e5d 100644 --- a/codegen/lang/processor.cpp +++ b/codegen/lang/processor.cpp @@ -11,6 +11,7 @@ #include "codegen/common/cpp_file.h" #include "codegen/lang/parsed_file.h" #include "codegen/lang/generator.h" +#include "codegen/lang/subsets.h" namespace codegen { namespace lang { @@ -18,6 +19,14 @@ namespace { constexpr int kErrorCantWritePath = 821; +[[nodiscard]] common::ProjectInfo Project(const QString &inputPath) { + return { + "codegen_style", + QFileInfo(inputPath).fileName(), + false, + }; +} + } // namespace Processor::Processor(const Options &options) @@ -26,6 +35,13 @@ Processor::Processor(const Options &options) } int Processor::launch() { + if (options_.subsetsOnly) { + return WriteSubsets( + options_.outputPath, + options_.sourcesPath, + Project(options_.inputPath)) ? 0 : -1; + } + if (!parser_->read()) { return -1; } @@ -38,24 +54,20 @@ int Processor::launch() { } bool Processor::write(const LangPack &langpack) const { - bool forceReGenerate = false; QDir dir(options_.outputPath); if (!dir.mkpath(".")) { common::logError(kErrorCantWritePath, "Command Line") << "can not open path for writing: " << dir.absolutePath().toStdString(); return false; } - QFileInfo srcFile(options_.inputPath); QString dstFilePath = dir.absolutePath() + "/lang_auto"; - common::ProjectInfo project = { - "codegen_style", - srcFile.fileName(), - forceReGenerate - }; + const auto project = Project(options_.inputPath); Generator generator(langpack, dstFilePath, project); if (!generator.writeHeader() + || !generator.writeCounts() + || !generator.writeAllKeys() || !generator.writeSource() || !common::TouchTimestamp(dstFilePath)) { return false; diff --git a/codegen/lang/subsets.cpp b/codegen/lang/subsets.cpp new file mode 100644 index 0000000..8c7df9d --- /dev/null +++ b/codegen/lang/subsets.cpp @@ -0,0 +1,526 @@ +// This file is part of Desktop App Toolkit, +// a set of libraries for developing nice desktop applications. +// +// For license and copyright information please follow this link: +// https://github.com/desktop-app/legal/blob/master/LEGAL +// +#include "codegen/lang/subsets.h" + +#include "codegen/common/logging.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace codegen { +namespace lang { +namespace { + +constexpr int kErrorCantReadKeys = 831; +constexpr int kErrorCantReadSource = 832; +constexpr int kErrorCantWriteSubset = 833; + +constexpr auto kCacheVersion = quint32(2); + +const auto kKeysFile = QString("lang_auto_keys.h"); +const auto kSubsetsFolder = QString("lang_subsets"); +const auto kCacheFile = QString("lang_subsets/.cache"); +const auto kDeclarationStart = QByteArray("inline constexpr phrase<"); +const auto kSpecializationStart = QByteArray("struct phrase<"); +const auto kPhraseUsage = QByteArray("phrase<"); + +[[nodiscard]] bool IsIdentifierChar(char ch) { + return (ch >= 'a' && ch <= 'z') + || (ch >= 'A' && ch <= 'Z') + || (ch >= '0' && ch <= '9') + || (ch == '_'); +} + +struct Declarations { + std::vector code; + QHash byName; + std::map specializations; + std::vector combinations; +}; + +struct Scanned { + qint64 modified = 0; + qint64 size = 0; + std::vector tokens; + std::vector combinations; + std::vector includes; +}; + +struct SourceFile { + QString absolute; + QString relative; + Scanned scanned; + std::vector keys; + std::vector included; + bool unit = false; + bool fresh = false; +}; + +struct Cache { + QHash scanned; + QHash subsets; + qint64 keysModified = 0; + qint64 keysSize = 0; +}; + +constexpr auto kSaneCount = qint32(1024 * 1024); + +QDataStream &operator<<(QDataStream &stream, const Scanned &value) { + const auto list = [&](const std::vector &values) { + stream << qint32(values.size()); + for (const auto &entry : values) { + stream << entry; + } + }; + stream << value.modified << value.size; + list(value.tokens); + list(value.combinations); + list(value.includes); + return stream; +} + +QDataStream &operator>>(QDataStream &stream, Scanned &value) { + const auto list = [&](std::vector &values) { + auto count = qint32(0); + stream >> count; + if (count < 0 || count > kSaneCount) { + stream.setStatus(QDataStream::ReadCorruptData); + return; + } + values.resize(count); + for (auto &entry : values) { + stream >> entry; + } + }; + stream >> value.modified >> value.size; + list(value.tokens); + list(value.combinations); + list(value.includes); + return stream; +} + +[[nodiscard]] QByteArray NormalizeCombination(const QByteArray &tags) { + auto result = QByteArrayList(); + for (const auto &tag : tags.split(',')) { + const auto trimmed = tag.trimmed(); + if (!trimmed.startsWith("lngtag_")) { + return QByteArray(); + } + result.push_back(trimmed); + } + return result.join(','); +} + +[[nodiscard]] bool ReadDeclarations( + const QString &path, + Declarations &result) { + auto file = QFile(path); + if (!file.open(QIODevice::ReadOnly)) { + common::logError(kErrorCantReadKeys, path) + << "can not open the generated keys header for reading"; + return false; + } + const auto content = file.readAll(); + file.close(); + + auto block = QByteArray(); + auto blockCombination = QByteArray(); + auto inside = false; + for (auto from = qsizetype(0); from < content.size();) { + auto till = content.indexOf('\n', from); + if (till < 0) { + till = content.size(); + } + const auto line = content.mid(from, till - from); + from = till + 1; + + if (inside) { + block += line + '\n'; + if (line == "};") { + result.specializations.emplace(blockCombination, block); + inside = false; + } + continue; + } else if (line.startsWith(kSpecializationStart)) { + const auto close = line.lastIndexOf('>'); + blockCombination = NormalizeCombination(line.mid( + kSpecializationStart.size(), + close - kSpecializationStart.size())); + block = "template <>\n" + line + '\n'; + inside = true; + continue; + } else if (!line.startsWith(kDeclarationStart)) { + continue; + } + const auto after = line.indexOf("> ", kDeclarationStart.size() - 1); + const auto brace = (after < 0) ? -1 : line.indexOf('{', after + 2); + if (after < 0 || brace < 0) { + common::logError(kErrorCantReadKeys, path) + << "bad key declaration: " << line.constData(); + return false; + } + result.byName.insert( + line.mid(after + 2, brace - after - 2).trimmed(), + int(result.code.size())); + result.combinations.push_back(NormalizeCombination(line.mid( + kDeclarationStart.size(), + after - kDeclarationStart.size()))); + result.code.push_back(line); + } + if (result.code.empty()) { + common::logError(kErrorCantReadKeys, path) + << "no key declarations found"; + return false; + } + return true; +} + +void Scan(const QByteArray &content, Scanned &result) { + const auto data = content.constData(); + const auto size = content.size(); + for (auto i = qsizetype(0); i + 4 < size; ++i) { + const auto languageKey = (data[i] == 'l' + && data[i + 1] == 'n' + && data[i + 2] == 'g') + || (data[i] == 'a' + && data[i + 1] == 'y' + && data[i + 2] == 'u'); + if (!languageKey + || data[i + 3] != '_' + || (i > 0 && IsIdentifierChar(data[i - 1]))) { + continue; + } + auto till = i + 4; + while (till != size && IsIdentifierChar(data[till])) { + ++till; + } + result.tokens.push_back(content.mid(i, till - i)); + i = till - 1; + } + for (auto from = qsizetype(0); from != size;) { + const auto found = content.indexOf(kPhraseUsage, from); + if (found < 0) { + break; + } + from = found + kPhraseUsage.size(); + const auto close = content.indexOf('>', from); + if (close < 0) { + break; + } + const auto combination = NormalizeCombination( + content.mid(from, close - from)); + if (!combination.isEmpty()) { + result.combinations.push_back(combination); + } + from = close + 1; + } + const auto directive = QByteArray("#include"); + for (auto from = qsizetype(0); from != size;) { + const auto found = content.indexOf(directive, from); + if (found < 0) { + break; + } + from = found + directive.size(); + + auto lineStart = found; + while (lineStart > 0 && data[lineStart - 1] != '\n') { + --lineStart; + } + auto clean = true; + for (auto i = lineStart; i != found; ++i) { + if (data[i] != ' ' && data[i] != '\t') { + clean = false; + break; + } + } + if (!clean) { + continue; + } + auto quote = from; + while (quote != size && (data[quote] == ' ' || data[quote] == '\t')) { + ++quote; + } + if (quote == size || data[quote] != '"') { + continue; + } + const auto till = content.indexOf('"', quote + 1); + if (till < 0) { + break; + } + result.includes.push_back(content.mid(quote + 1, till - quote - 1)); + from = till + 1; + } +} + +[[nodiscard]] Cache ReadCache(const QString &path) { + auto result = Cache(); + auto file = QFile(path); + if (!file.open(QIODevice::ReadOnly)) { + return result; + } + auto stream = QDataStream(&file); + auto version = quint32(0); + stream >> version; + if (version != kCacheVersion) { + return Cache(); + } + stream >> result.keysModified >> result.keysSize; + stream >> result.scanned >> result.subsets; + return (stream.status() == QDataStream::Ok) ? result : Cache(); +} + +void WriteCache(const QString &path, const Cache &cache) { + auto file = QFile(path); + if (!file.open(QIODevice::WriteOnly)) { + return; + } + auto stream = QDataStream(&file); + stream << kCacheVersion << cache.keysModified << cache.keysSize; + stream << cache.scanned << cache.subsets; +} + +[[nodiscard]] QByteArray Signature( + const std::vector &keys, + const std::set &combinations) { + auto result = QByteArray(); + for (const auto key : keys) { + result += QByteArray::number(key) + ' '; + } + result += '|'; + for (const auto &combination : combinations) { + result += combination + ' '; + } + return result; +} + +[[nodiscard]] bool WriteSubset( + const QString &path, + const Declarations &declarations, + const std::vector &keys, + const std::set &combinations, + const common::ProjectInfo &project) { + auto file = common::CppFile(path, project); + file.include("lang_auto.h").newline(); + file.pushNamespace("tr").newline(); + for (const auto &combination : combinations) { + const auto i = declarations.specializations.find(combination); + if (i != declarations.specializations.end()) { + file.stream() << i->second; + } + } + for (const auto index : keys) { + file.stream() << declarations.code[index] << "\n"; + } + file.newline().popNamespace(); + if (!file.finalize()) { + common::logError(kErrorCantWriteSubset, path) + << "can not write the lang keys subset"; + return false; + } + return true; +} + +} // namespace + +bool WriteSubsets( + const QString &genPath, + const QString &sourcesPath, + const common::ProjectInfo &project) { + const auto keysPath = genPath + '/' + kKeysFile; + const auto keysInfo = QFileInfo(keysPath); + auto cache = ReadCache(genPath + '/' + kCacheFile); + const auto keysSame = (cache.keysModified + == keysInfo.lastModified().toMSecsSinceEpoch()) + && (cache.keysSize == keysInfo.size()); + + const auto root = QDir(sourcesPath).absolutePath(); + auto files = std::vector(); + auto indices = QHash(); + auto iterator = QDirIterator( + root, + QDir::Files, + QDirIterator::Subdirectories); + while (iterator.hasNext()) { + const auto absolute = iterator.next(); + if (!absolute.endsWith(".cpp") + && !absolute.endsWith(".h") + && !absolute.endsWith(".mm")) { + continue; + } + const auto info = iterator.fileInfo(); + auto file = SourceFile(); + file.absolute = absolute; + file.relative = absolute.mid(root.size() + 1); + file.unit = absolute.endsWith(".cpp") || absolute.endsWith(".mm"); + file.scanned.modified = info.lastModified().toMSecsSinceEpoch(); + file.scanned.size = info.size(); + files.push_back(std::move(file)); + } + std::sort(files.begin(), files.end(), [](const auto &a, const auto &b) { + return a.relative < b.relative; + }); + for (auto i = 0, count = int(files.size()); i != count; ++i) { + indices.insert(files[i].absolute, i); + } + + auto dirty = !keysSame || (int(cache.scanned.size()) != int(files.size())); + for (auto &file : files) { + const auto i = cache.scanned.constFind(file.relative); + if (i != cache.scanned.cend() + && i->modified == file.scanned.modified + && i->size == file.scanned.size) { + file.scanned = *i; + continue; + } + auto device = QFile(file.absolute); + if (!device.open(QIODevice::ReadOnly)) { + common::logError(kErrorCantReadSource, file.absolute) + << "can not open the source file for reading"; + return false; + } + auto scanned = Scanned(); + scanned.modified = file.scanned.modified; + scanned.size = file.scanned.size; + Scan(device.readAll(), scanned); + if (i == cache.scanned.cend() + || i->tokens != scanned.tokens + || i->combinations != scanned.combinations + || i->includes != scanned.includes) { + dirty = true; + } + file.scanned = std::move(scanned); + } + + const auto subsets = genPath + '/' + kSubsetsFolder; + if (!dirty) { + auto complete = true; + for (const auto &file : files) { + if (file.unit + && !QFileInfo::exists(subsets + '/' + file.relative + ".h")) { + complete = false; + break; + } + } + if (complete) { + auto next = Cache(); + next.keysModified = keysInfo.lastModified().toMSecsSinceEpoch(); + next.keysSize = keysInfo.size(); + next.subsets = cache.subsets; + for (const auto &file : files) { + next.scanned.insert(file.relative, file.scanned); + } + WriteCache(genPath + '/' + kCacheFile, next); + return true; + } + } + + auto declarations = Declarations(); + if (!ReadDeclarations(keysPath, declarations)) { + return false; + } + + for (auto &file : files) { + for (const auto &token : file.scanned.tokens) { + const auto i = declarations.byName.constFind(token); + if (i != declarations.byName.cend()) { + file.keys.push_back(*i); + } + } + const auto dir = file.absolute.left(file.absolute.lastIndexOf('/')); + for (const auto &include : file.scanned.includes) { + const auto relative = QString::fromUtf8(include); + const auto exact = !relative.contains("..") + && !relative.contains("./"); + for (const auto &base : { dir, root }) { + const auto joined = base + '/' + relative; + const auto i = indices.constFind( + exact ? joined : QDir::cleanPath(joined)); + if (i != indices.cend()) { + file.included.push_back(*i); + break; + } + } + } + } + + auto next = Cache(); + next.keysModified = keysInfo.lastModified().toMSecsSinceEpoch(); + next.keysSize = keysInfo.size(); + + auto visited = std::vector(files.size(), -1); + auto taken = std::vector(declarations.code.size(), -1); + auto queue = std::vector(); + auto keys = std::vector(); + auto combinations = std::set(); + for (auto i = 0, count = int(files.size()); i != count; ++i) { + next.scanned.insert(files[i].relative, files[i].scanned); + if (!files[i].unit) { + continue; + } + keys.clear(); + combinations.clear(); + queue.clear(); + queue.push_back(i); + visited[i] = i; + for (auto j = 0; j != int(queue.size()); ++j) { + const auto &file = files[queue[j]]; + for (const auto key : file.keys) { + if (taken[key] != i) { + taken[key] = i; + keys.push_back(key); + const auto &combination = declarations.combinations[key]; + if (!combination.isEmpty()) { + combinations.emplace(combination); + } + } + } + for (const auto &combination : file.scanned.combinations) { + combinations.emplace(combination); + } + for (const auto included : file.included) { + if (visited[included] != i) { + visited[included] = i; + queue.push_back(included); + } + } + } + std::sort(keys.begin(), keys.end()); + const auto signature = Signature(keys, combinations); + next.subsets.insert(files[i].relative, signature); + + const auto path = subsets + '/' + files[i].relative + ".h"; + const auto known = cache.subsets.constFind(files[i].relative); + if (keysSame + && known != cache.subsets.cend() + && *known == signature + && QFileInfo::exists(path)) { + continue; + } else if (!WriteSubset( + path, + declarations, + keys, + combinations, + project)) { + return false; + } + } + WriteCache(genPath + '/' + kCacheFile, next); + return true; +} + +} // namespace lang +} // namespace codegen diff --git a/codegen/lang/subsets.h b/codegen/lang/subsets.h new file mode 100644 index 0000000..4ca425d --- /dev/null +++ b/codegen/lang/subsets.h @@ -0,0 +1,21 @@ +// This file is part of Desktop App Toolkit, +// a set of libraries for developing nice desktop applications. +// +// For license and copyright information please follow this link: +// https://github.com/desktop-app/legal/blob/master/LEGAL +// +#pragma once + +#include +#include "codegen/common/cpp_file.h" + +namespace codegen { +namespace lang { + +bool WriteSubsets( + const QString &genPath, + const QString &sourcesPath, + const common::ProjectInfo &project); + +} // namespace lang +} // namespace codegen diff --git a/codegen/style/generator.cpp b/codegen/style/generator.cpp index f0baac6..3756811 100644 --- a/codegen/style/generator.cpp +++ b/codegen/style/generator.cpp @@ -304,7 +304,7 @@ QString Generator::valueAssignmentCode( const QString& name) const { auto copy = value.copyOf(); if (!ignoreCopy && !copy.isEmpty()) { - return "st::" + copy.back(); + return "st::" + copy.join('.'); } switch (value.type().tag) { diff --git a/codegen/style/parsed_file.cpp b/codegen/style/parsed_file.cpp index 7462a44..3963efd 100644 --- a/codegen/style/parsed_file.cpp +++ b/codegen/style/parsed_file.cpp @@ -773,7 +773,8 @@ structure::Value ParsedFile::readFontValue() { structure::Value family, size; do { if (auto formatToken = file_.getToken(BasicType::Name)) { - if (tokenValue(formatToken) == "bold") { + if (tokenValue(formatToken) == "bold" + || tokenValue(formatToken) == "semibold") { flags |= structure::data::font::Bold; continue; } else if (tokenValue(formatToken) == "italic") { @@ -782,9 +783,6 @@ structure::Value ParsedFile::readFontValue() { } else if (tokenValue(formatToken) == "underline") { flags |= structure::data::font::Underline; continue; - } else if (tokenValue(formatToken) == "semibold") { - flags |= structure::data::font::Semibold; - continue; } else { file_.putBack(); } @@ -851,6 +849,7 @@ structure::Value ParsedFile::readCopyValue() { structure::FullName name = { tokenValue(copyName) }; if (auto variable = module_->findVariable(name)) { auto result = variable->value; + auto copyOf = variable->name; while (file_.getToken(BasicType::Dot)) { if (auto fieldName = file_.getToken(BasicType::Name)) { auto fieldNameStr = tokenValue(fieldName); @@ -900,6 +899,7 @@ structure::Value ParsedFile::readCopyValue() { for (const auto &field : *fields) { if (field.variable.name == fieldFullName) { result = field.variable.value; + copyOf.push_back(fieldNameStr); found = true; break; } @@ -916,7 +916,7 @@ structure::Value ParsedFile::readCopyValue() { return {}; } } - return result.makeCopy(variable->name); + return result.makeCopy(copyOf); } file_.putBack(); } diff --git a/codegen/style/structure_types.h b/codegen/style/structure_types.h index af8e419..683529d 100644 --- a/codegen/style/structure_types.h +++ b/codegen/style/structure_types.h @@ -81,7 +81,6 @@ struct font { Bold = 0x01, Italic = 0x02, Underline = 0x04, - Semibold = 0x10, }; std::string family; int size;