Sync codegen with Telegram Desktop 7.0.1

codegen/lang/CMakeLists.txt

@@ -14,6 +14,8 @@ PRIVATE
     codegen/lang/parsed_file.h
     codegen/lang/processor.cpp
     codegen/lang/processor.h
+    codegen/lang/subsets.cpp
+    codegen/lang/subsets.h
 )
 
 target_include_directories(codegen_lang

codegen/lang/generator.cpp

@@ -6,42 +6,195 @@
 //
 #include "codegen/lang/generator.h"
 
+#include <algorithm>
+#include <limits>
 #include <memory>
 #include <functional>
 #include <QtCore/QDir>
+#include <QtCore/QFile>
 #include <QtCore/QSet>
+#include <QtCore/QTextStream>
 #include <QtGui/QImage>
 #include <QtGui/QPainter>
 
 namespace codegen {
 namespace lang {
+namespace {
+
+constexpr int kErrorTooManyKeys = 841;
+constexpr int kErrorTooManyTags = 842;
+
+constexpr auto kIndexLimit = std::numeric_limits<ushort>::max();
+
+struct Slot {
+	int start = 0;
+	int size = 0;
+};
+
+} // namespace
 
 Generator::Generator(const LangPack &langpack, const QString &destBasePath, const common::ProjectInfo &project)
 : langpack_(langpack)
 , basePath_(destBasePath)
 , baseName_(QFileInfo(basePath_).baseName())
 , project_(project) {
+	allocateIndices();
+	saveTagOrder();
+	collectDeclarations();
+}
+
+void Generator::saveTagOrder() {
+	auto file = QFile(basePath_ + ".tags");
+	if (file.open(QIODevice::WriteOnly)) {
+		auto stream = QTextStream(&file);
+		for (const auto &tag : langpack_.tags) {
+			stream << tag.tag << '\n';
+		}
+	}
+}
+
+void Generator::allocateIndices() {
+	const auto path = basePath_ + ".indices";
+	auto known = std::map<QString, Slot>();
+	auto next = 0;
+
+	auto reading = QFile(path);
+	if (reading.open(QIODevice::ReadOnly)) {
+		auto stream = QTextStream(&reading);
+		while (!stream.atEnd()) {
+			const auto parts = stream.readLine().split('\t');
+			if (parts.size() != 3) {
+				continue;
+			}
+			const auto start = parts[1].toInt();
+			const auto size = parts[2].toInt();
+			if (size <= 0 || start < 0) {
+				continue;
+			}
+			known.emplace(parts[0], Slot{ start, size });
+			next = std::max(next, start + size);
+		}
+		reading.close();
+	}
+
+	indices_.assign(langpack_.entries.size(), -1);
+	for (auto i = 0, count = int(langpack_.entries.size()); i != count; ++i) {
+		const auto &entry = langpack_.entries[i];
+		const auto isPlural = !entry.keyBase.isEmpty();
+		if (isPlural && entry.key != ComputePluralKey(entry.keyBase, 0)) {
+			continue; // Taken by the first part of the plural group.
+		}
+		const auto name = isPlural ? entry.keyBase : entry.key;
+		const auto size = isPlural ? kPluralPartCount : 1;
+		auto j = known.find(name);
+		if (j == known.end() || j->second.size != size) {
+			j = known.insert_or_assign(name, Slot{ next, size }).first;
+			next += size;
+		}
+		for (auto shift = 0; shift != size; ++shift) {
+			indices_[i + shift] = j->second.start + shift;
+		}
+	}
+	keysCount_ = next;
+	if (keysCount_ > kIndexLimit) {
+		common::logError(kErrorTooManyKeys, path)
+			<< "the key table needs " << keysCount_
+			<< " slots, the limit is " << kIndexLimit
+			<< ". Removed keys keep their slot so that re-adding one is free; "
+			<< "delete this file to renumber the keys from scratch.";
+		failed_ = true;
+		return;
+	}
+
+	auto writing = QFile(path);
+	if (writing.open(QIODevice::WriteOnly)) {
+		auto stream = QTextStream(&writing);
+		for (const auto &[name, slot] : known) {
+			stream << name << '\t' << slot.start << '\t' << slot.size << '\n';
+		}
+	}
+}
+
+void Generator::collectDeclarations() {
+	auto byIndex = std::map<int, QString>();
+	for (auto i = 0, count = int(langpack_.entries.size()); i != count; ++i) {
+		const auto &entry = langpack_.entries[i];
+		const auto isPlural = !entry.keyBase.isEmpty();
+		const auto &key = entry.key;
+		if (isPlural && key != ComputePluralKey(entry.keyBase, 0)) {
+			continue;
+		}
+		auto tags = QStringList();
+		for (auto &tagData : entry.tags) {
+			tags.push_back("lngtag_" + tagData.tag);
+		}
+		byIndex.emplace(indices_[i], "inline constexpr phrase<"
+			+ tags.join(", ")
+			+ "> "
+			+ (isPlural ? entry.keyBase : key)
+			+ "{ ushort("
+			+ QString::number(indices_[i])
+			+ ") };");
+	}
+	declarations_.reserve(byIndex.size());
+	for (auto &[index, declaration] : byIndex) {
+		declarations_.push_back(declaration);
+	}
 }
 
 bool Generator::writeHeader() {
+	if (failed_) {
+		return false;
+	} else if (int(langpack_.tags.size()) > kIndexLimit) {
+		common::logError(kErrorTooManyTags, basePath_ + ".tags")
+			<< "there are " << langpack_.tags.size()
+			<< " tags, the limit is " << kIndexLimit
+			<< ". Delete this file to drop the tags that are not used anymore.";
+		return false;
+	}
 	header_ = std::make_unique<common::CppFile>(basePath_ + ".h", project_);
 	header_->include("lang/lang_tag.h").include("lang/lang_values.h").newline();
 
-	writeHeaderForwardDeclarations();
 	writeHeaderTagTypes();
 	writeHeaderInterface();
 	writeHeaderReactiveInterface();
+	writeHeaderKeysInclude();
 
 	return header_->finalize();
 }
 
-void Generator::writeHeaderForwardDeclarations() {
-	header_->pushNamespace("Lang").stream() << "\
+void Generator::writeHeaderKeysInclude() {
+	header_->stream() << "\
+#ifdef LANG_KEYS_SUBSET\n\
+#include LANG_KEYS_SUBSET\n\
+#else // LANG_KEYS_SUBSET\n\
+#include \"" << baseName_ << "_keys.h\"\n\
+#endif // LANG_KEYS_SUBSET\n";
+}
+
+bool Generator::writeCounts() {
+	auto file = common::CppFile(basePath_ + "_counts.h", project_);
+	file.pushNamespace("Lang").stream() << "\
 \n\
 inline constexpr auto kTagsCount = ushort(" << langpack_.tags.size() << ");\n\
-inline constexpr auto kKeysCount = ushort(" << langpack_.entries.size() << ");\n\
+inline constexpr auto kKeysCount = ushort(" << keysCount_ << ");\n\
 \n";
-	header_->popNamespace().newline();
+	file.popNamespace();
+	return file.finalize();
+}
+
+bool Generator::writeAllKeys() {
+	auto file = common::CppFile(basePath_ + "_keys.h", project_);
+	file.include(baseName_ + ".h").newline();
+	file.pushNamespace("tr").newline();
+	for (const auto &[combination, code] : specializations_) {
+		file.stream() << code;
+	}
+	for (const auto &declaration : declarations_) {
+		file.stream() << declaration << "\n";
+	}
+	file.newline().popNamespace();
+	return file.finalize();
 }
 
 void Generator::writeHeaderTagTypes() {
@@ -98,7 +251,6 @@ void Generator::writeHeaderReactiveInterface() {
 	header_->pushNamespace("tr");
 
 	writeHeaderProducersInterface();
-	writeHeaderProducersInstances();
 
 	header_->popNamespace().newline();
 }
@@ -132,7 +284,7 @@ using S = std::decay_t<decltype(std::declval<P>()(QString()))>;\n\
 template <typename ...Tags>\n\
 struct phrase;\n\
 \n";
-	std::set<QString> producersDeclared;
+	auto specializations = std::map<QString, QString>();
 	for (auto &entry : langpack_.entries) {
 		const auto isPlural = !entry.keyBase.isEmpty();
 		auto tags = QStringList();
@@ -160,58 +312,62 @@ struct phrase;\n\
 		}
 		producerArgs.push_back("P p = P()");
 		currentArgs.push_back("P p = P()");
-		if (!producersDeclared.emplace(tags.join(',')).second) {
+		const auto combination = tags.join(',');
+		if (specializations.find(combination) != specializations.end()) {
 			continue;
 		}
-		header_->stream() << "\
+		specializations.emplace(combination, "\
 template <>\n\
-struct phrase<" << tags.join(", ") << "> {\n\
+struct phrase<" + tags.join(", ") + "> {\n\
 	template <typename P = details::Identity>\n\
-	rpl::producer<S<P>> operator()(" << producerArgs.join(", ") << ") const {\n\
-		return ::Lang::details::Producer<" << tags.join(", ") << ">::Combine(" << values.join(", ") << ");\n\
+	rpl::producer<S<P>> operator()(" + producerArgs.join(", ") + ") const {\n\
+		return ::Lang::details::Producer<" + tags.join(", ") + ">::Combine(" + values.join(", ") + ");\n\
 	}\n\
 \n\
 	template <typename P = details::Identity>\n\
-	S<P> operator()(now_t, " << currentArgs.join(", ") << ") const {\n\
-		return ::Lang::details::Producer<" << tags.join(", ") << ">::Current(" << values.join(", ") << ");\n\
+	S<P> operator()(now_t, " + currentArgs.join(", ") + ") const {\n\
+		return ::Lang::details::Producer<" + tags.join(", ") + ">::Current(" + values.join(", ") + ");\n\
 	}\n\
 \n\
 	ushort base;\n\
 };\n\
-\n";
+\n");
 	}
-}
-
-void Generator::writeHeaderProducersInstances() {
-	auto index = 0;
-	for (auto &entry : langpack_.entries) {
-		const auto isPlural = !entry.keyBase.isEmpty();
-		const auto &key = entry.key;
-		auto tags = QStringList();
-		for (auto &tagData : entry.tags) {
-			const auto &tag = tagData.tag;
-			tags.push_back("lngtag_" + tag);
-		}
-		if (!isPlural || key == ComputePluralKey(entry.keyBase, 0)) {
-			header_->stream() << "\
-inline constexpr phrase<" << tags.join(", ") << "> " << (isPlural ? entry.keyBase : key) << "{ ushort(" << index << ") };\n";
-		}
-		++index;
+	const auto plain = specializations.find(QString());
+	if (plain != specializations.end()) {
+		header_->stream() << plain->second;
+		specializations.erase(plain);
 	}
-	header_->newline();
+	specializations_ = std::move(specializations);
 }
 
 bool Generator::writeSource() {
 	source_ = std::make_unique<common::CppFile>(basePath_ + ".cpp", project_);
 
-	source_->include("lang/lang_keys.h").pushNamespace("Lang").pushNamespace();
+	source_->include("lang/lang_keys.h")
+		.include(baseName_ + "_counts.h")
+		.pushNamespace("Lang")
+		.pushNamespace();
+
+	source_->stream() << "\
+static_assert(sizeof(QChar) == sizeof(char16_t));\n\
+static_assert(alignof(QChar) == alignof(char16_t));\n\
+\n";
+
+	auto byIndex = std::vector<const LangPack::Entry*>(keysCount_, nullptr);
+	for (auto i = 0, count = int(langpack_.entries.size()); i != count; ++i) {
+		byIndex[indices_[i]] = &langpack_.entries[i];
+	}
 
 	source_->stream() << "\
-QChar DefaultData[] = {";
+const char16_t DefaultData[] = {";
 	auto count = 0;
 	auto fulllength = 0;
-	for (auto &entry : langpack_.entries) {
-		for (auto ch : entry.value) {
+	for (const auto entry : byIndex) {
+		if (!entry) {
+			continue;
+		}
+		for (auto ch : entry->value) {
 			if (fulllength > 0) source_->stream() << ",";
 			if (!count++) {
 				source_->stream() << "\n";
@@ -221,7 +377,7 @@ QChar DefaultData[] = {";
 				}
 				source_->stream() << " ";
 			}
-			source_->stream() << "QChar(0x" << QString::number(ch.unicode(), 16) << ")";
+			source_->stream() << "0x" << QString::number(ch.unicode(), 16);
 			++fulllength;
 		}
 	}
@@ -230,8 +386,9 @@ QChar DefaultData[] = {";
 int Offsets[] = {";
 	count = 0;
 	auto offset = 0;
-	auto writeOffset = [this, &count, &offset] {
-		if (offset > 0) source_->stream() << ",";
+	auto written = 0;
+	auto writeOffset = [this, &count, &offset, &written] {
+		if (written++ > 0) source_->stream() << ",";
 		if (!count++) {
 			source_->stream() << "\n";
 		} else {
@@ -242,9 +399,9 @@ int Offsets[] = {";
 		}
 		source_->stream() << offset;
 	};
-	for (auto &entry : langpack_.entries) {
+	for (const auto entry : byIndex) {
 		writeOffset();
-		offset += entry.value.size();
+		offset += entry ? entry->value.size() : 0;
 	}
 	writeOffset();
 	source_->stream() << " };\n";
@@ -270,10 +427,11 @@ ushort GetKeyIndex(QLatin1String key) {\n\
 	auto size = key.size();\n\
 	auto data = key.data();\n";
 
-	auto index = 0;
 	auto indices = std::map<QString, QString>();
-	for (auto &entry : langpack_.entries) {
-		indices.emplace(getFullKey(entry), QString::number(index++));
+	for (auto i = 0, total = int(langpack_.entries.size()); i != total; ++i) {
+		indices.emplace(
+			getFullKey(langpack_.entries[i]),
+			QString::number(indices_[i]));
 	}
 	const auto indexOfKey = [&](const QString &full) {
 		const auto i = indices.find(full);
@@ -354,7 +512,9 @@ QString GetOriginalValue(ushort key) {\n\
 	Expects(key < kKeysCount);\n\
 \n\
 	const auto offset = Offsets[key];\n\
-	return QString::fromRawData(DefaultData + offset, Offsets[key + 1] - offset);\n\
+	return QString::fromRawData(\n\
+		reinterpret_cast<const QChar*>(DefaultData + offset),\n\
+		Offsets[key + 1] - offset);\n\
 }\n\
 \n";