Add language metadata and enhance language validation tests (#2748)

Resolves #2739

## Description:

Introduce language metadata handling and refactor existing language
checks to validate the existence of language JSON and corresponding SVG
files. Add tests to ensure the integrity of the new metadata structure
and its references.

The lang field is intentionally kept in each language file.
This is because the files are frequently regenerated by Crowdin, and the
field also serves as a hint for management and maintenance.

## Please complete the following:

- [x] I have added screenshots for all UI updates
- [x] I process any text displayed to the user through translateText()
and I've added it to the en.json file
- [x] I have added relevant tests to the test directory
- [x] I confirm I have thoroughly tested these changes and take full
responsibility for any bugs introduced

## Please put your Discord username so you can be contacted if a bug or
regression is found:

aotumuri

---------

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>

resources/lang/metadata.json

@@ -0,0 +1,206 @@
+[
+  {
+    "code": "ar",
+    "native": "العربية",
+    "en": "Arabic",
+    "svg": "ps"
+  },
+  {
+    "code": "bg",
+    "native": "Български",
+    "en": "Bulgarian",
+    "svg": "bg"
+  },
+  {
+    "code": "bn",
+    "native": "বাংলা",
+    "en": "Bengali",
+    "svg": "bd"
+  },
+  {
+    "code": "cs",
+    "native": "Čeština",
+    "en": "Czech",
+    "svg": "cz"
+  },
+  {
+    "code": "da",
+    "native": "Dansk",
+    "en": "Danish",
+    "svg": "dk"
+  },
+  {
+    "code": "de",
+    "native": "Deutsch",
+    "en": "German",
+    "svg": "de"
+  },
+  {
+    "code": "el",
+    "native": "Ελληνικά",
+    "en": "Greek",
+    "svg": "gr"
+  },
+  {
+    "code": "en",
+    "native": "English",
+    "en": "English",
+    "svg": "uk_us_flag"
+  },
+  {
+    "code": "eo",
+    "native": "Esperanto",
+    "en": "Esperanto",
+    "svg": "eo"
+  },
+  {
+    "code": "es",
+    "native": "Español",
+    "en": "Spanish",
+    "svg": "es"
+  },
+  {
+    "code": "fa",
+    "native": "فارسی",
+    "en": "Persian",
+    "svg": "ir"
+  },
+  {
+    "code": "fi",
+    "native": "suomi",
+    "en": "Finnish",
+    "svg": "fi"
+  },
+  {
+    "code": "fr",
+    "native": "Français",
+    "en": "French",
+    "svg": "fr"
+  },
+  {
+    "code": "gl",
+    "native": "Galego",
+    "en": "Galician",
+    "svg": "es-ga"
+  },
+  {
+    "code": "he",
+    "native": "עברית",
+    "en": "Hebrew",
+    "svg": "il"
+  },
+  {
+    "code": "hi",
+    "native": "हिन्दी",
+    "en": "Hindi",
+    "svg": "in"
+  },
+  {
+    "code": "hu",
+    "native": "Magyar",
+    "en": "Hungarian",
+    "svg": "hu"
+  },
+  {
+    "code": "it",
+    "native": "Italiano",
+    "en": "Italian",
+    "svg": "it"
+  },
+  {
+    "code": "ja",
+    "native": "日本語",
+    "en": "Japanese",
+    "svg": "jp"
+  },
+  {
+    "code": "ko",
+    "native": "한국어",
+    "en": "Korean",
+    "svg": "kr"
+  },
+  {
+    "code": "mk",
+    "native": "Македонски",
+    "en": "Macedonian",
+    "svg": "mk"
+  },
+  {
+    "code": "nl",
+    "native": "Nederlands",
+    "en": "Dutch",
+    "svg": "nl"
+  },
+  {
+    "code": "pl",
+    "native": "Polski",
+    "en": "Polish",
+    "svg": "pl"
+  },
+  {
+    "code": "pt-BR",
+    "native": "Português brasileiro",
+    "en": "Brazilian Portuguese",
+    "svg": "br"
+  },
+  {
+    "code": "pt-PT",
+    "native": "Português",
+    "en": "European Portuguese",
+    "svg": "pt"
+  },
+  {
+    "code": "ru",
+    "native": "Русский",
+    "en": "Russian",
+    "svg": "ru"
+  },
+  {
+    "code": "sh",
+    "native": "Srpsko-Hrvatski",
+    "en": "Serbo-Croatian",
+    "svg": "sh_yugo"
+  },
+  {
+    "code": "sk",
+    "native": "Slovenčina",
+    "en": "Slovak",
+    "svg": "sk"
+  },
+  {
+    "code": "sl",
+    "native": "Slovenščina",
+    "en": "Slovenian",
+    "svg": "si"
+  },
+  {
+    "code": "sv-SE",
+    "native": "Svenska",
+    "en": "Swedish",
+    "svg": "se"
+  },
+  {
+    "code": "tp",
+    "native": "toki pona",
+    "en": "Toki Pona",
+    "svg": "toki_pona"
+  },
+  {
+    "code": "tr",
+    "native": "Türkçe",
+    "en": "Turkish",
+    "svg": "tr"
+  },
+  {
+    "code": "uk",
+    "native": "Українська",
+    "en": "Ukrainian",
+    "svg": "ua"
+  },
+  {
+    "code": "zh-CN",
+    "native": "简体中文",
+    "en": "Chinese Simplified",
+    "svg": "cn"
+  }
+]

src/client/LangSelector.ts

@@ -2,40 +2,15 @@ import { LitElement, html } from "lit";
 import { customElement, state } from "lit/decorators.js";
 import "./LanguageModal";
 
-import ar from "../../resources/lang/ar.json";
-import bg from "../../resources/lang/bg.json";
-import bn from "../../resources/lang/bn.json";
-import cs from "../../resources/lang/cs.json";
-import da from "../../resources/lang/da.json";
-import de from "../../resources/lang/de.json";
-import el from "../../resources/lang/el.json";
 import en from "../../resources/lang/en.json";
-import eo from "../../resources/lang/eo.json";
-import es from "../../resources/lang/es.json";
-import fa from "../../resources/lang/fa.json";
-import fi from "../../resources/lang/fi.json";
-import fr from "../../resources/lang/fr.json";
-import gl from "../../resources/lang/gl.json";
-import he from "../../resources/lang/he.json";
-import hi from "../../resources/lang/hi.json";
-import hu from "../../resources/lang/hu.json";
-import it from "../../resources/lang/it.json";
-import ja from "../../resources/lang/ja.json";
-import ko from "../../resources/lang/ko.json";
-import mk from "../../resources/lang/mk.json";
-import nl from "../../resources/lang/nl.json";
-import pl from "../../resources/lang/pl.json";
-import pt_BR from "../../resources/lang/pt-BR.json";
-import pt_PT from "../../resources/lang/pt-PT.json";
-import ru from "../../resources/lang/ru.json";
-import sh from "../../resources/lang/sh.json";
-import sk from "../../resources/lang/sk.json";
-import sl from "../../resources/lang/sl.json";
-import sv_SE from "../../resources/lang/sv-SE.json";
-import tp from "../../resources/lang/tp.json";
-import tr from "../../resources/lang/tr.json";
-import uk from "../../resources/lang/uk.json";
-import zh_CN from "../../resources/lang/zh-CN.json";
+import metadata from "../../resources/lang/metadata.json";
+
+type LanguageMetadata = {
+  code: string;
+  native: string;
+  en: string;
+  svg: string;
+};
 
 @customElement("lang-selector")
 export class LangSelector extends LitElement {
@@ -47,43 +22,8 @@ export class LangSelector extends LitElement {
   @state() private debugMode: boolean = false;
 
   private debugKeyPressed: boolean = false;
-
-  private languageMap: Record<string, any> = {
-    ar,
-    bg,
-    bn,
-    de,
-    el,
-    en,
-    es,
-    eo,
-    fr,
-    it,
-    hi,
-    hu,
-    ja,
-    nl,
-    pl,
-    "pt-PT": pt_PT,
-    "pt-BR": pt_BR,
-    ru,
-    sh,
-    tr,
-    tp,
-    uk,
-    cs,
-    he,
-    da,
-    fa,
-    fi,
-    "sv-SE": sv_SE,
-    "zh-CN": zh_CN,
-    ko,
-    mk,
-    gl,
-    sl,
-    sk,
-  };
+  private languageMetadata: LanguageMetadata[] = metadata;
+  private languageCache = new Map<string, Record<string, string>>();
 
   createRenderRoot() {
     return this;
@@ -106,10 +46,12 @@ export class LangSelector extends LitElement {
 
   private getClosestSupportedLang(lang: string): string {
     if (!lang) return "en";
-    if (lang in this.languageMap) return lang;
+    if (lang === "debug") return "debug";
+    const supported = new Set(this.languageMetadata.map((entry) => entry.code));
+    if (supported.has(lang)) return lang;
 
     const base = lang.slice(0, 2);
-    const candidates = Object.keys(this.languageMap).filter((key) =>
+    const candidates = Array.from(supported).filter((key) =>
       key.startsWith(base),
     );
     if (candidates.length > 0) {
@@ -125,41 +67,53 @@ export class LangSelector extends LitElement {
     const savedLang = localStorage.getItem("lang");
     const userLang = this.getClosestSupportedLang(savedLang ?? browserLocale);
 
-    this.defaultTranslations = this.loadLanguage("en");
-    this.translations = this.loadLanguage(userLang);
+    const [defaultTranslations, translations] = await Promise.all([
+      this.loadLanguage("en"),
+      this.loadLanguage(userLang),
+    ]);
+
+    this.defaultTranslations = defaultTranslations;
+    this.translations = translations;
     this.currentLang = userLang;
 
     await this.loadLanguageList();
     this.applyTranslation();
   }
 
-  private loadLanguage(lang: string): Record<string, string> {
-    const language = this.languageMap[lang] ?? {};
-    const flat = flattenTranslations(language);
-    return flat;
+  private async loadLanguage(lang: string): Promise<Record<string, string>> {
+    if (!lang) return {};
+    const cached = this.languageCache.get(lang);
+    if (cached) return cached;
+
+    if (lang === "en") {
+      const flat = flattenTranslations(en);
+      this.languageCache.set(lang, flat);
+      return flat;
+    }
+
+    try {
+      const response = await fetch(`/lang/${encodeURIComponent(lang)}.json`);
+      if (!response.ok) {
+        throw new Error(`Failed to fetch language ${lang}: ${response.status}`);
+      }
+      const language = (await response.json()) as Record<string, any>;
+      const flat = flattenTranslations(language);
+      this.languageCache.set(lang, flat);
+      return flat;
+    } catch (err) {
+      console.error(`Failed to load language ${lang}:`, err);
+      return {};
+    }
   }
 
   private async loadLanguageList() {
     try {
-      const data = this.languageMap;
       let list: any[] = [];
 
       const browserLang = new Intl.Locale(navigator.language).language;
 
-      for (const langCode of Object.keys(data)) {
-        const langData = data[langCode].lang;
-        if (!langData) continue;
-
-        list.push({
-          code: langData.lang_code ?? langCode,
-          native: langData.native ?? langCode,
-          en: langData.en ?? langCode,
-          svg: langData.svg ?? langCode,
-        });
-      }
-
       let debugLang: any = null;
-      if (this.debugKeyPressed) {
+      if (this.debugKeyPressed || this.currentLang === "debug") {
         debugLang = {
           code: "debug",
           native: "Debug",
@@ -169,6 +123,16 @@ export class LangSelector extends LitElement {
         this.debugMode = true;
       }
 
+      for (const langData of this.languageMetadata) {
+        if (langData.code === "debug" && !debugLang) continue;
+        list.push({
+          code: langData.code,
+          native: langData.native,
+          en: langData.en,
+          svg: langData.svg,
+        });
+      }
+
       const currentLangEntry = list.find((l) => l.code === this.currentLang);
       const browserLangEntry =
         browserLang !== this.currentLang && browserLang !== "en"
@@ -202,9 +166,9 @@ export class LangSelector extends LitElement {
     }
   }
 
-  private changeLanguage(lang: string) {
+  private async changeLanguage(lang: string) {
     localStorage.setItem("lang", lang);
-    this.translations = this.loadLanguage(lang);
+    this.translations = await this.loadLanguage(lang);
     this.currentLang = lang;
     this.applyTranslation();
     this.showModal = false;
@@ -277,10 +241,10 @@ export class LangSelector extends LitElement {
     return text;
   }
 
-  private openModal() {
+  private async openModal() {
     this.debugMode = this.debugKeyPressed;
     this.showModal = true;
-    this.loadLanguageList();
+    await this.loadLanguageList();
   }
 
   render() {

tests/LangCode.test.ts

@@ -1,27 +0,0 @@
-import fs from "fs";
-import path from "path";
-
-describe("LangCode Filename Check", () => {
-  const langDir = path.join(__dirname, "../resources/lang");
-
-  test("lang_code matches filename", () => {
-    const files = fs
-      .readdirSync(langDir)
-      .filter((file) => file.endsWith(".json"));
-
-    if (files.length === 0) {
-      console.log("No resources/lang/*.json files found. Skipping check.");
-      return;
-    }
-
-    for (const file of files) {
-      const filePath = path.join(langDir, file);
-      const jsonData = JSON.parse(fs.readFileSync(filePath, "utf-8"));
-
-      const fileNameWithoutExt = path.basename(file, ".json");
-      const langCode = jsonData.lang?.lang_code;
-
-      expect(fileNameWithoutExt).toBe(langCode);
-    }
-  });
-});

tests/LangMetadata.test.ts

@@ -0,0 +1,62 @@
+import fs from "fs";
+import path from "path";
+
+describe("Lang Metadata Check", () => {
+  const langDir = path.join(__dirname, "../resources/lang");
+  const flagDir = path.join(__dirname, "../resources/flags");
+  const metadataFile = path.join(langDir, "metadata.json");
+
+  test("metadata languages point to existing lang json and flag files", () => {
+    if (!fs.existsSync(metadataFile)) {
+      console.log(
+        "No resources/lang/metadata.json file found. Skipping check.",
+      );
+      return;
+    }
+
+    const metadata = JSON.parse(fs.readFileSync(metadataFile, "utf-8"));
+    if (!Array.isArray(metadata) || metadata.length === 0) {
+      console.log(
+        "No language entries found in metadata.json. Skipping check.",
+      );
+      return;
+    }
+
+    const errors: string[] = [];
+
+    for (const entry of metadata) {
+      const code = entry?.code;
+      const svg = entry?.svg;
+      if (typeof code !== "string" || code.length === 0) {
+        errors.push(
+          `metadata entry missing valid code: ${JSON.stringify(entry)}`,
+        );
+        continue;
+      }
+      if (typeof svg !== "string" || svg.length === 0) {
+        errors.push(
+          `[${code}]: metadata svg is missing or not a non-empty string`,
+        );
+        continue;
+      }
+
+      const langFilePath = path.join(langDir, `${code}.json`);
+      if (!fs.existsSync(langFilePath)) {
+        errors.push(`[${code}]: lang json file does not exist: ${code}.json`);
+      }
+
+      const svgFile = svg.endsWith(".svg") ? svg : `${svg}.svg`;
+      const flagPath = path.join(flagDir, svgFile);
+      if (!fs.existsSync(flagPath)) {
+        errors.push(`[${code}]: SVG file does not exist: ${svgFile}`);
+      }
+    }
+
+    if (errors.length > 0) {
+      console.error(
+        "Metadata lang or SVG file check failed:\n" + errors.join("\n"),
+      );
+      expect(errors).toEqual([]);
+    }
+  });
+});

tests/LangSvg.test.ts

@@ -1,53 +0,0 @@
-import fs from "fs";
-import path from "path";
-
-describe("Lang SVG Field and File Existence Check", () => {
-  const langDir = path.join(__dirname, "../resources/lang");
-  const flagDir = path.join(__dirname, "../resources/flags");
-
-  test("each lang.json file has a valid lang.svg string and the SVG file exists", () => {
-    const files = fs
-      .readdirSync(langDir)
-      .filter((file) => file.endsWith(".json"));
-
-    if (files.length === 0) {
-      console.log("No resources/lang/*.json files found. Skipping check.");
-      return;
-    }
-
-    const errors: string[] = [];
-
-    for (const file of files) {
-      try {
-        const filePath = path.join(langDir, file);
-        const jsonData = JSON.parse(fs.readFileSync(filePath, "utf-8"));
-        const langSvg = jsonData.lang?.svg;
-        if (typeof langSvg !== "string" || langSvg.length === 0) {
-          errors.push(
-            `[${file}]: lang.svg is missing or not a non-empty string`,
-          );
-          continue;
-        }
-
-        // Check if the SVG file exists in the flags directory
-        const svgFile = langSvg.endsWith(".svg") ? langSvg : `${langSvg}.svg`;
-        const flagPath = path.join(flagDir, svgFile);
-
-        if (!fs.existsSync(flagPath)) {
-          errors.push(`[${file}]: SVG file does not exist: ${svgFile}`);
-        }
-      } catch (err) {
-        errors.push(
-          `[${file}]: Exception occurred - ${(err as Error).message}`,
-        );
-      }
-    }
-
-    if (errors.length > 0) {
-      console.error(
-        "Lang SVG field or file check failed:\n" + errors.join("\n"),
-      );
-      expect(errors).toEqual([]);
-    }
-  });
-});