Add statistics

src/atoms/data.ts

@@ -1,6 +1,13 @@
 import { atom, useAtomValue } from "jotai";
 import { primitiveRepertoireAtom } from "./constants";
-import { CustomReadings, isPUA } from "~/lib";
+import {
+  CharacterSetSpecifier,
+  CustomReadings,
+  PrimitiveCharacter,
+  isPUA,
+  isValidCJKBasicChar,
+  isValidCJKChar,
+} from "~/lib";
 import { recursiveRenderCompound } from "~/lib";
 import { dataAtom } from ".";
 import { focusAtom } from "jotai-optics";
@@ -9,6 +16,9 @@ import { CustomGlyph } from "~/lib";
 import { determine } from "~/lib";
 import { classifier } from "~/lib";
 
+export const characterSetAtom = focusAtom(dataAtom, (o) =>
+  o.prop("character_set").valueOr("general" as CharacterSetSpecifier),
+);
 export const userRepertoireAtom = focusAtom(dataAtom, (o) =>
   o.prop("repertoire").valueOr({} as PrimitiveRepertoire),
 );
@@ -25,6 +35,25 @@ export const userTagsAtom = focusAtom(dataAtom, (o) =>
   o.prop("tags").valueOr([] as string[]),
 );
 
+export const charactersAtom = atom((get) => {
+  const primitiveRepertoire = get(primitiveRepertoireAtom);
+  const characterSet = get(characterSetAtom);
+  const filters: Record<
+    CharacterSetSpecifier,
+    (k: string, v: PrimitiveCharacter) => boolean
+  > = {
+    general: (_, v) => v.tygf > 0,
+    basic: (k, v) => v.tygf > 0 || isValidCJKBasicChar(k),
+    extended: (k, v) => v.tygf > 0 || isValidCJKChar(k),
+  };
+
+  const filter = filters[characterSet];
+  const characters = Object.entries(primitiveRepertoire)
+    .filter(([k, v]) => filter(k, v))
+    .map(([k]) => k);
+  return characters;
+});
+
 export const allRepertoireAtom = atom((get) => {
   const repertoire = get(primitiveRepertoireAtom);
   const userRepertoire = get(userRepertoireAtom);
@@ -53,10 +82,11 @@ export const glyphAtom = atom((get) => {
   const result = new Map<string, SVGGlyph>();
   for (const [char, { glyph }] of Object.entries(repertoire)) {
     if (glyph === undefined) continue;
+    if (result.has(char)) continue;
     if (glyph.type === "basic_component") {
       result.set(char, glyph.strokes);
     } else {
-      const svgglyph = recursiveRenderCompound(glyph, repertoire);
+      const svgglyph = recursiveRenderCompound(glyph, repertoire, result);
       if (svgglyph instanceof Error) continue;
       result.set(char, svgglyph);
     }

src/components/Debugger.tsx

@@ -3,17 +3,21 @@ import { ColumnsType } from "antd/es/table";
 import { useAtom, useAtomValue } from "jotai";
 import { atomWithStorage } from "jotai/utils";
 import { useMemo, useState } from "react";
-import { DictEntry, EncodeResult, configAtom, repertoireAtom } from "~/atoms";
+import {
+  DictEntry,
+  EncodeResult,
+  charactersAtom,
+  configAtom,
+  repertoireAtom,
+} from "~/atoms";
 import { encodeResultAtom } from "~/atoms/cache";
 import { Select, Uploader } from "~/components/Utils";
 import { getSupplemental } from "~/lib";
 
 export default function Debugger() {
   const config = useAtomValue(configAtom);
   const repertoire = useAtomValue(repertoireAtom);
-  const characters = Object.entries(repertoire)
-    .filter(([, v]) => v.tygf > 0)
-    .map(([x]) => x);
+  const characters = useAtomValue(charactersAtom);
   const code = useAtomValue(encodeResultAtom) ?? [];
   const referenceAtom = useMemo(
     () =>

src/components/Optimizer.tsx

@@ -17,6 +17,7 @@ import {
   dictionaryAtom,
   DictEntry,
   makeEncodeCallback,
+  charactersAtom,
 } from "~/atoms";
 import {
   assemble,
@@ -86,9 +87,7 @@ export default function Optimizer() {
   const [analysisResult, setAnalysisResult] = useAtom(analysisResultAtom);
   const [assemblyResult, setAssemblyResult] = useAtom(assemblyResultAtom);
   const repertoire = useAtomValue(repertoireAtom);
-  const characters = Object.entries(repertoire)
-    .filter(([_, v]) => v.tygf > 0)
-    .map(([x]) => x);
+  const characters = useAtomValue(charactersAtom);
   const customElements = useAtomValue(customElementsAtom);
   const [out1, setOut1] = useState("");
   const [result, setResult] = useState<[Date, string][]>([]);

src/components/SequenceTable.tsx

@@ -13,6 +13,7 @@ import {
   useSetAtom,
   makeEncodeCallback,
   DictEntry,
+  charactersAtom,
 } from "~/atoms";
 import type { Assembly, IndexedElement } from "~/lib";
 import {
@@ -21,16 +22,13 @@ import {
   stringifySequence,
   summarize,
   analysis,
-  defaultAnalyzer,
-  analyzePrimitiveDuplication,
 } from "~/lib";
 import { exportTSV, makeWorker, renderIndexed, renderSuperScript } from "~/lib";
 import {
   analysisResultAtom,
   assemblyResultAtom,
   encodeResultAtom,
 } from "~/atoms/cache";
-import PrimitiveDuplicationAnalyzer from "~/components/PrimitiveDuplicationAnalyzer";
 import { ProColumns, ProTable } from "@ant-design/pro-components";
 import ProrityShortCodeSelector from "./ProrityShortCodeSelector";
 import { customElementsAtom } from "~/atoms/assets";
@@ -50,9 +48,7 @@ interface MainEntry {
 const RecomputeAssembly = () => {
   const repertoire = useAtomValue(repertoireAtom);
   const config = useAtomValue(configAtom);
-  const characters = Object.entries(repertoire)
-    .filter(([, v]) => v.tygf > 0)
-    .map(([x]) => x);
+  const characters = useAtomValue(charactersAtom);
   const [analysisResult, setAnalysisResult] = useAtom(analysisResultAtom);
   const [assemblyResult, setAssemblyResult] = useAtom(assemblyResultAtom);
   const customElements = useAtomValue(customElementsAtom);
@@ -199,31 +195,25 @@ export default function SequenceTable() {
     ...encodeResult[i]!,
   }));
 
-  const [analyzer, setAnalyzer] = useState(defaultAnalyzer);
-  const [selections, filtered] = analyzePrimitiveDuplication(
-    analyzer,
-    frequencyMap,
-    combinedResult,
+  const dataSource = combinedResult.map(
+    ({ name, sequence, importance, ...rest }) => {
+      const frequency = Math.round(
+        ((frequencyMap[name] ?? 0) * importance) / 100,
+      );
+      const key = `${name}-${summarize(sequence)}`;
+      const entry: MainEntry = {
+        key,
+        frequency,
+        name,
+        ...rest,
+      };
+      for (const [i, element] of sequence.entries()) {
+        entry[i] = element;
+      }
+      return entry;
+    },
   );
 
-  const toShow = analyzer.filter ? filtered : combinedResult;
-  const dataSource = toShow.map(({ name, sequence, importance, ...rest }) => {
-    const frequency = Math.round(
-      ((frequencyMap[name] ?? 0) * importance) / 100,
-    );
-    const key = `${name}-${summarize(sequence)}`;
-    const entry: MainEntry = {
-      key,
-      frequency,
-      name,
-      ...rest,
-    };
-    for (const [i, element] of sequence.entries()) {
-      entry[i] = element;
-    }
-    return entry;
-  });
-
   dataSource.sort((a, b) => b.frequency - a.frequency);
 
   const hash = (record: MainEntry) => {
@@ -372,12 +362,6 @@ export default function SequenceTable() {
           closable
         />
       ) : null}
-      {assemblyResult.length > 0 && (
-        <PrimitiveDuplicationAnalyzer
-          selections={selections}
-          setAnalyzer={setAnalyzer}
-        />
-      )}
       <ProTable<MainEntry>
         virtual
         scroll={{ y: 1080 }}

src/components/Utils.tsx

@@ -11,24 +11,11 @@ import {
   notification,
 } from "antd";
 import styled from "styled-components";
-import {
-  getRecordFromTSV,
-  getDictFromTSV,
-  getDistributionFromTSV,
-} from "~/lib";
-import { useEffect, useState } from "react";
+import { useState } from "react";
 import DeleteOutlined from "@ant-design/icons/DeleteOutlined";
 import PlusOutlined from "@ant-design/icons/PlusOutlined";
 import MinusOutlined from "@ant-design/icons/MinusOutlined";
 import Root from "./Element";
-import {
-  defaultDictionaryAtom,
-  fetchAsset,
-  frequencyAtom,
-  keyDistributionAtom,
-  pairEquivalenceAtom,
-} from "~/atoms";
-import { useSetAtom } from "jotai";
 
 const ScrollableRow = styled(Row)`
   height: 100%;
@@ -164,17 +151,3 @@ export const KeyList = ({
     </Flex>
   );
 };
-
-export function LoadAssets() {
-  const setF = useSetAtom(frequencyAtom);
-  const setW = useSetAtom(defaultDictionaryAtom);
-  const setKE = useSetAtom(keyDistributionAtom);
-  const setPE = useSetAtom(pairEquivalenceAtom);
-  fetchAsset("frequency", "txt").then((x) => setF(getRecordFromTSV(x)));
-  fetchAsset("dictionary", "txt").then((x) => setW(getDictFromTSV(x)));
-  fetchAsset("key_distribution", "txt").then((x) =>
-    setKE(getDistributionFromTSV(x)),
-  );
-  fetchAsset("pair_equivalence", "txt").then((x) => setPE(getRecordFromTSV(x)));
-  return null;
-}

src/lib/assembly.ts

@@ -368,9 +368,10 @@ export const stringifySequence = (result: AssemblyResult, config: Config) => {
   });
 };
 
-export const summarize = (elements: IndexedElement[]) => {
+export const summarize = (elements: (IndexedElement | undefined)[]) => {
   return elements
     .map((x) => {
+      if (x === undefined) return "ε";
       if (typeof x === "string") return x;
       else if (x.index === 0) {
         return x.element;