Skip to content

hsg.classes

KnownSet

hsg.classes.knownset

KnownSet

ABC for 'which characters does the learner know'.

Backends: Heisig (frame range), HSK (level cap), file (user list), anki-export (Anki deck export).

get_char_info(char)

Return backend-specific metadata for a character.

Heisig returns {frame, keyword, pinyin, frequency}. HSK returns {level}. File returns {char}. Override in subclasses.

Source code in hsg/classes/knownset.py 
28
29
30
31
32
33
34
35
def get_char_info(self, char: str) -> dict[str, Any]:
    """Return backend-specific metadata for a character.

    Heisig returns {frame, keyword, pinyin, frequency}.
    HSK returns {level}. File returns {char}.
    Override in subclasses.
    """
    return {'char': char}

get_known_characters() abstractmethod

Return the full list of known characters (excluding ADDITIONAL).

Source code in hsg/classes/knownset.py 
19
20
21
22
@abstractmethod
def get_known_characters(self) -> list[str]:
    """Return the full list of known characters (excluding ADDITIONAL)."""
    raise NotImplementedError

get_statistics(chars)

Compute coverage statistics for a list of characters.

This is a concrete method — it only calls is_known(), so it works for any KnownSet backend.

Source code in hsg/classes/knownset.py 
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
def get_statistics(self, chars: list[str]) -> dict[str, Any]:
    """Compute coverage statistics for a list of characters.

    This is a concrete method — it only calls is_known(), so it
    works for any KnownSet backend.
    """
    unique_chars = [c for i, c in enumerate(chars) if c not in chars[:i]]
    total_chars = len(chars)
    total_chars_unique = len(unique_chars)
    total_known = len([c for c in chars if self.is_known(c)])
    total_known_unique = len([c for c in unique_chars if self.is_known(c)])
    total_known_percent = round(total_known / total_chars * 100, 2) if total_chars > 0 else 0
    total_known_unique_percent = (
        round(total_known_unique / total_chars_unique * 100, 2) if total_chars_unique > 0 else 0
    )
    frequencies: dict[str, dict[str, Any]] = {}
    for c in chars:
        if c not in frequencies:
            frequencies[c] = {'occurrencies': 1, 'percent': None}
        else:
            frequencies[c]['occurrencies'] += 1
    for c in frequencies:
        frequencies[c]['percent'] = round(frequencies[c]['occurrencies'] / len(chars) * 100, 2)
    return {
        'chars': total_chars,
        'known': total_known,
        'known_percent': total_known_percent,
        'unknown': total_chars - total_known,
        'unknown_percent': round(100 - total_known_percent, 2),
        'chars_unique': total_chars_unique,
        'known_unique': total_known_unique,
        'known_unique_percent': total_known_unique_percent,
        'unknown_unique': total_chars_unique - total_known_unique,
        'unknown_unique_percent': round(100 - total_known_unique_percent, 2),
        'frequencies': frequencies,
    }

is_additional_character(char)

Return True if char is a non-Hanzi allowlist character.

Source code in hsg/classes/knownset.py 
24
25
26
def is_additional_character(self, char: str) -> bool:
    """Return True if char is a non-Hanzi allowlist character."""
    return char in ADDITIONAL_CHARACTERS

is_known(char) abstractmethod

Return True if the learner knows this character.

Source code in hsg/classes/knownset.py 
14
15
16
17
@abstractmethod
def is_known(self, char: str) -> bool:
    """Return True if the learner knows this character."""
    raise NotImplementedError

Heisig

hsg.classes.heisig

Heisig(frequencies_corpus, maxframe=-1)

Bases: KnownSet

KnownSet backed by Heisig Remembering the Hanzi (RSH) frame data.

Loads frame data from assets/heisig.tsv. A character is 'known' if its frame number <= maxframe. Also provides frequency-enriched frame metadata (keyword, pinyin, frequency rank) via the configured Frequency corpus.

Initialise with a frequency corpus name and optional frame cap (-1 = all).

Source code in hsg/classes/heisig.py 
22
23
24
25
26
27
28
def __init__(self, frequencies_corpus: str, maxframe: int = -1) -> None:
    """Initialise with a frequency corpus name and optional frame cap (-1 = all)."""
    self.maxframe = maxframe
    self.frequencies: Frequency = create_frequency(frequencies_corpus)
    self.heisig: dict[str, dict[str, Any]] = {}
    self.load_heisig()
    self.known_characters: list[str] = self.get_known_characters()

get_char_info(char)

Return the full frame metadata dict for a character.

Source code in hsg/classes/heisig.py 
69
70
71
def get_char_info(self, char: str) -> dict[str, Any]:
    """Return the full frame metadata dict for a character."""
    return self.heisig[char]

get_frame_info(char)

Alias for get_char_info (kept for backward compatibility).

Source code in hsg/classes/heisig.py 
73
74
75
def get_frame_info(self, char: str) -> dict[str, Any]:
    """Alias for get_char_info (kept for backward compatibility)."""
    return self.get_char_info(char)

get_known_characters()

Return known frames plus ADDITIONAL_CHARACTERS.

Source code in hsg/classes/heisig.py 
61
62
63
def get_known_characters(self) -> list[str]:
    """Return known frames plus ADDITIONAL_CHARACTERS."""
    return self.get_known_frames() + list(ADDITIONAL_CHARACTERS)

get_known_frames()

Return hanzi whose frame number <= maxframe.

Source code in hsg/classes/heisig.py 
57
58
59
def get_known_frames(self) -> list[str]:
    """Return hanzi whose frame number <= maxframe."""
    return [hanzi for hanzi in self.heisig if self.heisig[hanzi]['frame'] <= self.maxframe]

is_known(char)

Return True if char is in the known set (frame <= maxframe or additional).

Source code in hsg/classes/heisig.py 
65
66
67
def is_known(self, char: str) -> bool:
    """Return True if char is in the known set (frame <= maxframe or additional)."""
    return char in self.known_characters

load_heisig()

Parse heisig.tsv into self.heisig, enriching each entry with frequency data.

Source code in hsg/classes/heisig.py 
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
def load_heisig(self) -> None:
    """Parse heisig.tsv into self.heisig, enriching each entry with frequency data."""
    with open(HEISIG_CSV) as f:
        reader = csv.reader(f, delimiter='\t')
        for row in reader:
            frame = row[2]
            if frame.startswith('v') or not frame:
                continue
            frame_num = int(frame)
            hanzi = row[0]
            keyword = row[4]
            pinyin = row[5]
            frequency_data = self.frequencies.find_char(hanzi)
            self.heisig[hanzi] = {
                'hanzi': hanzi,
                'frame': frame_num,
                'keyword': keyword,
                'pinyin': pinyin,
                'frequency': frequency_data['rank'] if frequency_data else 9999,
            }
    if self.maxframe == -1:
        self.maxframe = max([f['frame'] for f in self.heisig.values()])

output(words, format)

Print frame data in the specified format (csv, json, tabulate).

Source code in hsg/classes/heisig.py 
77
78
79
80
81
82
83
84
85
86
87
88
89
def output(self, words: list[dict[str, Any]], format: str) -> None:
    """Print frame data in the specified format (csv, json, tabulate)."""
    if format == 'csv':
        fields = ('hanzi', 'frame', 'keyword', 'pinyin', 'frequency')
        writer: csv.DictWriter[str] = csv.DictWriter(
            sys.stdout, fieldnames=fields, delimiter='\t', extrasaction='ignore'
        )
        writer.writeheader()
        writer.writerows(words)
    elif format == 'json':
        print(json.dumps(words))
    elif format == 'tabulate':
        print(tabulate(words, headers='keys', tablefmt='github'))

set_max_frame(maxframe)

Update the frame cap after construction.

Source code in hsg/classes/heisig.py 
30
31
32
def set_max_frame(self, maxframe: int) -> None:
    """Update the frame cap after construction."""
    self.maxframe = maxframe

HSK

hsg.classes.hsk

HSK()

HSK vocabulary data reader (both HSK 2.0 old and HSK 3.0 new lists).

Loads word and character data from hsk_old.csv and hsk_new.csv. Provides level lookups for words and individual characters.

Load both HSK old and new lists.

Source code in hsg/classes/hsk.py 
15
16
17
18
19
20
21
22
def __init__(self) -> None:
    """Load both HSK old and new lists."""
    self.hsk_old: OrderedDict[str, dict[str, Any]] = OrderedDict()
    self.hsk_new: OrderedDict[str, dict[str, Any]] = OrderedDict()
    self.hsk_old_chars: OrderedDict[str, str] = OrderedDict()
    self.hsk_new_chars: OrderedDict[str, str] = OrderedDict()
    self.load_old_hsk(HSK_OLD_CSV)
    self.load_new_hsk(HSK_NEW_CSV)

get_hsk_new_char_level(char)

Return the HSK 3.0 level for a single character, or None.

Source code in hsg/classes/hsk.py 
 98
 99
100
def get_hsk_new_char_level(self, char: str) -> str | None:
    """Return the HSK 3.0 level for a single character, or None."""
    return self.hsk_new_chars.get(char)

get_hsk_new_chars(level=None)

Return HSK 3.0 characters at the given level (None = all).

Source code in hsg/classes/hsk.py 
108
109
110
111
112
def get_hsk_new_chars(self, level: str | int | None = None) -> list[str]:
    """Return HSK 3.0 characters at the given level (None = all)."""
    if level:
        return [c for c in self.hsk_new_chars if self.hsk_new_chars[c] == str(level)]
    return list(self.hsk_new_chars.keys())

get_hsk_new_word(char)

Look up a word in the HSK 3.0 list. Returns None if not found.

Source code in hsg/classes/hsk.py 
64
65
66
def get_hsk_new_word(self, char: str) -> dict[str, Any] | None:
    """Look up a word in the HSK 3.0 list. Returns None if not found."""
    return self.hsk_new.get(char)

get_hsk_new_word_level(word)

Return the HSK 3.0 level for a word, or None.

Source code in hsg/classes/hsk.py 
87
88
89
90
91
92
def get_hsk_new_word_level(self, word: str) -> str | None:
    """Return the HSK 3.0 level for a word, or None."""
    if word in self.hsk_new:
        level: str = self.hsk_new[word]['level']
        return level
    return None

get_hsk_new_words(level)

Return HSK 3.0 words at the given level (None = all levels).

Source code in hsg/classes/hsk.py 
74
75
76
77
78
def get_hsk_new_words(self, level: str | int | None) -> list[dict[str, Any]]:
    """Return HSK 3.0 words at the given level (None = all levels)."""
    if level:
        return [w for w in self.hsk_new.values() if w['level'] == str(level)]
    return list(self.hsk_new.values())

get_hsk_old_char_level(char)

Return the HSK 2.0 level for a single character, or None.

Source code in hsg/classes/hsk.py 
94
95
96
def get_hsk_old_char_level(self, char: str) -> str | None:
    """Return the HSK 2.0 level for a single character, or None."""
    return self.hsk_old_chars.get(char)

get_hsk_old_chars(level=None)

Return HSK 2.0 characters at the given level (None = all).

Source code in hsg/classes/hsk.py 
102
103
104
105
106
def get_hsk_old_chars(self, level: str | int | None = None) -> list[str]:
    """Return HSK 2.0 characters at the given level (None = all)."""
    if level:
        return [c for c in self.hsk_old_chars if self.hsk_old_chars[c] == str(level)]
    return list(self.hsk_old_chars.keys())

get_hsk_old_word(char)

Look up a word in the HSK 2.0 list. Returns None if not found.

Source code in hsg/classes/hsk.py 
60
61
62
def get_hsk_old_word(self, char: str) -> dict[str, Any] | None:
    """Look up a word in the HSK 2.0 list. Returns None if not found."""
    return self.hsk_old.get(char)

get_hsk_old_word_level(word)

Return the HSK 2.0 level for a word, or None.

Source code in hsg/classes/hsk.py 
80
81
82
83
84
85
def get_hsk_old_word_level(self, word: str) -> str | None:
    """Return the HSK 2.0 level for a word, or None."""
    if word in self.hsk_old:
        level: str = self.hsk_old[word]['level']
        return level
    return None

get_hsk_old_words(level)

Return HSK 2.0 words at the given level (None = all levels).

Source code in hsg/classes/hsk.py 
68
69
70
71
72
def get_hsk_old_words(self, level: str | int | None) -> list[dict[str, Any]]:
    """Return HSK 2.0 words at the given level (None = all levels)."""
    if level:
        return [w for w in self.hsk_old.values() if w['level'] == str(level)]
    return list(self.hsk_old.values())

load_new_hsk(newhskcsv)

Parse the HSK 3.0 word list into self.hsk_new.

Source code in hsg/classes/hsk.py 
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
def load_new_hsk(self, newhskcsv: str) -> None:
    """Parse the HSK 3.0 word list into self.hsk_new."""
    with open(newhskcsv) as f:
        reader = csv.reader(f, delimiter=',')
        next(reader)
        for row in reader:
            level, num, simplified, pinyin, definitions, *_rest = row
            self.hsk_new[simplified] = {
                'level': level,
                'num': int(num),
                'simplified': simplified,
                'pinyin': pinyin,
                'translations': definitions,
            }
            for c in simplified:
                if c not in ADDITIONAL_CHARACTERS and c not in self.hsk_new_chars:
                    self.hsk_new_chars[c] = level

load_old_hsk(oldhskcsv)

Parse the HSK 2.0 word list into self.hsk_old.

Source code in hsg/classes/hsk.py 
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
def load_old_hsk(self, oldhskcsv: str) -> None:
    """Parse the HSK 2.0 word list into self.hsk_old."""
    with open(oldhskcsv) as f:
        reader = csv.reader(f, delimiter='\t')
        next(reader)
        for row in reader:
            level, num, simplified, pinyin, translations = row
            self.hsk_old[simplified] = {
                'level': level,
                'num': int(num),
                'simplified': simplified,
                'pinyin': pinyin,
                'translations': translations.replace(',', ';'),
            }
            for c in simplified:
                if c not in ADDITIONAL_CHARACTERS and c not in self.hsk_old_chars:
                    self.hsk_old_chars[c] = level

HSKKnownSet

hsg.classes.hsk_knownset

HSKKnownSet(max_level=6, use_old=False)

Bases: KnownSet

KnownSet backed by HSK levels.

A character is 'known' if its HSK new-list level <= max_level, or if it's in ADDITIONAL_CHARACTERS.

Source code in hsg/classes/hsk_knownset.py 
29
30
31
32
33
def __init__(self, max_level: int = 6, use_old: bool = False) -> None:
    self.hsk = HSK()
    self.max_level = max_level
    self.use_old = use_old
    self._known_chars: set[str] = set(self._compute_known_chars())

FileKnownSet

hsg.classes.file_knownset

FileKnownSet(filepath)

Bases: KnownSet

KnownSet backed by a user-supplied file (one character per line).

Source code in hsg/classes/file_knownset.py 
10
11
12
13
14
15
16
17
def __init__(self, filepath: str) -> None:
    self.filepath = filepath
    self._chars: set[str] = set()
    with open(filepath) as f:
        for line in f:
            char = line.strip()
            if char and len(char) == 1:
                self._chars.add(char)

Frequency

hsg.classes.frequency

Frequency

ABC for character/word frequency corpora.

Backends: SubtlexCh (subtitle-based), RenMinWang (newspaper-based).

find_char(char) abstractmethod

Look up frequency data for a single character. Returns None if not found.

Source code in hsg/classes/frequency.py 
15
16
17
18
@abstractmethod
def find_char(self, char: str) -> dict[str, Any] | None:
    """Look up frequency data for a single character. Returns None if not found."""
    raise NotImplementedError

find_word(word) abstractmethod

Look up frequency data for a multi-character word. Returns None if not found.

Source code in hsg/classes/frequency.py 
20
21
22
23
@abstractmethod
def find_word(self, word: str) -> dict[str, Any] | None:
    """Look up frequency data for a multi-character word. Returns None if not found."""
    raise NotImplementedError

get_most_frequent_lemmas(type='chars', num=-1, skip_known=None, only_known=None, min_length=1, sort='rank', reverse=False)

Return ranked lemmas, optionally filtered by known-set membership.

Parameters:

Name Type Description Default
type str

'chars' or 'words'.

 'chars'
num int

Max results (-1 = all).

 -1
skip_known set[str] | None

Exclude these characters.

 None
only_known set[str] | None

Include only these characters.

 None
min_length int

Minimum lemma length.

 1
sort str

Sort key ('rank' or 'frequency').

 'rank'
reverse bool

Descending order if True.

 False
Source code in hsg/classes/frequency.py 
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
def get_most_frequent_lemmas(
    self,
    type: str = 'chars',
    num: int = -1,
    skip_known: set[str] | None = None,
    only_known: set[str] | None = None,
    min_length: int = 1,
    sort: str = 'rank',
    reverse: bool = False,
) -> list[dict[str, Any]]:
    """Return ranked lemmas, optionally filtered by known-set membership.

    Args:
        type: 'chars' or 'words'.
        num: Max results (-1 = all).
        skip_known: Exclude these characters.
        only_known: Include only these characters.
        min_length: Minimum lemma length.
        sort: Sort key ('rank' or 'frequency').
        reverse: Descending order if True.
    """
    raise NotImplementedError

RenMinWang

hsg.classes.renminwang

RenMinWang()

Bases: Frequency

Frequency corpus based on Renminwang (People's Daily) newspaper data.

Provides character and word frequency data from the renminwang/ assets.

Load character and word frequency CSVs.

Source code in hsg/classes/renminwang.py 
14
15
16
17
18
19
def __init__(self) -> None:
    """Load character and word frequency CSVs."""
    self.char_freq: list[dict[str, Any]] = self.load_csv(RMW_FREQUENCIES_CHARS_CSV)
    self.word_freq: list[dict[str, Any]] = self.load_csv(RMW_FREQUENCIES_WORDS_CSV)
    self.chars: dict[str, Any] = self.create_dict(self.char_freq)
    self.words: dict[str, Any] = self.create_dict(self.word_freq)

create_dict(lemmas)

Build a lemma-to-record lookup dict from a list.

Source code in hsg/classes/renminwang.py 
42
43
44
45
46
47
def create_dict(self, lemmas: list[dict[str, Any]]) -> dict[str, Any]:
    """Build a lemma-to-record lookup dict from a list."""
    result: dict[str, Any] = {}
    for lemma in lemmas:
        result[lemma['lemma']] = lemma
    return result

find_char(char)

Look up frequency data for a character. Returns None if not found.

Source code in hsg/classes/renminwang.py 
49
50
51
def find_char(self, char: str) -> dict[str, Any] | None:
    """Look up frequency data for a character. Returns None if not found."""
    return self.chars.get(char)

find_word(word)

Look up frequency data for a word. Returns None if not found.

Source code in hsg/classes/renminwang.py 
53
54
55
def find_word(self, word: str) -> dict[str, Any] | None:
    """Look up frequency data for a word. Returns None if not found."""
    return self.words.get(word)

get_most_frequent_lemmas(type='chars', num=-1, skip_known=None, only_known=None, min_length=1, sort='rank', reverse=False)

Return ranked lemmas, optionally filtered by known-set membership.

Source code in hsg/classes/renminwang.py 
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
def get_most_frequent_lemmas(
    self,
    type: str = 'chars',
    num: int = -1,
    skip_known: set[str] | None = None,
    only_known: set[str] | None = None,
    min_length: int = 1,
    sort: str = 'rank',
    reverse: bool = False,
) -> list[dict[str, Any]]:
    """Return ranked lemmas, optionally filtered by known-set membership."""
    lemmas = self.char_freq if type == 'chars' else self.word_freq
    if num == -1:
        num = len(lemmas)
    if skip_known is not None:
        lemmas = [lemma for lemma in lemmas if lemma['lemma'] not in skip_known]
    if only_known is not None:
        lemmas = [lemma for lemma in lemmas if lemma['lemma'] in only_known]
    data = sorted(lemmas, key=lambda x: x[sort], reverse=reverse)
    return data[:num]

load_csv(csvfile)

Parse a Renminwang frequency TSV, skipping header lines.

Source code in hsg/classes/renminwang.py 
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
def load_csv(self, csvfile: str) -> list[dict[str, Any]]:
    """Parse a Renminwang frequency TSV, skipping header lines."""
    with open(csvfile) as f:
        fields = (
            'lemma',
            'count',
            'count_million',
            'count_log',
            'cd',
            'cd_percent',
            'cd_log',
            'rank',
            'count_x_cd',
        )
        reader = csv.DictReader(f, fieldnames=fields, delimiter='\t')
        reader_no_headers = list(reader)[3:]  # skip first 3 lines
        for idx, lemma in enumerate(reader_no_headers):
            lemma['rank'] = idx + 1
            lemma['count_x_cd'] = int(lemma['count']) * int(lemma['cd'])
        return reader_no_headers

SubtlexCh

hsg.classes.subtlexch

SubtlexCh()

Bases: Frequency

Frequency corpus based on SUBTLEX-CH (Chinese film subtitle) data.

Provides character and word frequency, including part-of-speech data, from the subtlex-ch/ assets. Source: https://www.ugent.be/pp/experimentele-psychologie/en/research/documents/subtlexch/

Load character, word, and word+POS frequency CSVs.

Source code in hsg/classes/subtlexch.py 
90
91
92
93
94
95
96
97
def __init__(self) -> None:
    """Load character, word, and word+POS frequency CSVs."""
    self.char_freq: list[dict[str, Any]] = self.load_csv(SUBTLEX_CH_CHARS_CSV, self.FIELDS)
    self.word_freq: list[dict[str, Any]] = self.load_csv(SUBTLEX_CH_WORDS_CSV, self.FIELDS)
    self.word_pos_freq: list[dict[str, Any]] = self.load_csv(SUBTLEX_CH_WORDS_POS_COMBINED_CSV, self.FIELDS_POS)
    self.chars: dict[str, Any] = self.create_dict(self.char_freq)
    self.words: dict[str, Any] = self.create_dict(self.word_freq)
    self.words_pos: dict[str, Any] = self.create_dict(self.word_pos_freq)

create_dict(lemmas)

Build a lemma-to-record lookup dict from a list.

Source code in hsg/classes/subtlexch.py 
109
110
111
112
113
114
def create_dict(self, lemmas: list[dict[str, Any]]) -> dict[str, Any]:
    """Build a lemma-to-record lookup dict from a list."""
    result: dict[str, Any] = {}
    for lemma in lemmas:
        result[lemma['lemma']] = lemma
    return result

find_char(char)

Look up frequency data for a character. Returns None if not found.

Source code in hsg/classes/subtlexch.py 
116
117
118
def find_char(self, char: str) -> dict[str, Any] | None:
    """Look up frequency data for a character. Returns None if not found."""
    return self.chars.get(char)

find_pos(word)

Return the dominant part of speech and POS frequency counts for a word.

Source code in hsg/classes/subtlexch.py 
124
125
126
127
128
129
130
131
132
def find_pos(self, word: str) -> tuple[str, dict[str, int]] | None:
    """Return the dominant part of speech and POS frequency counts for a word."""
    data = self.words_pos.get(word)
    if data:
        all_pos = [p for p in data['all_pos'].split('.') if p]
        all_pos_freq = [int(f) for f in data['all_pos_freq'].split('.') if f]
        all_pos_dict = {all_pos[i]: all_pos_freq[i] for i in range(len(all_pos))}
        return (data['dominant_pos'], all_pos_dict)
    return None

find_word(word)

Look up frequency data for a word. Returns None if not found.

Source code in hsg/classes/subtlexch.py 
120
121
122
def find_word(self, word: str) -> dict[str, Any] | None:
    """Look up frequency data for a word. Returns None if not found."""
    return self.words.get(word)

get_most_frequent_lemmas(type='chars', num=-1, skip_known=None, only_known=None, min_length=1, sort='rank', reverse=False)

Return ranked lemmas, optionally filtered by known-set membership.

Source code in hsg/classes/subtlexch.py 
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
def get_most_frequent_lemmas(
    self,
    type: str = 'chars',
    num: int = -1,
    skip_known: set[str] | None = None,
    only_known: set[str] | None = None,
    min_length: int = 1,
    sort: str = 'rank',
    reverse: bool = False,
) -> list[dict[str, Any]]:
    """Return ranked lemmas, optionally filtered by known-set membership."""
    lemmas = self.char_freq if type == 'chars' else self.word_freq
    if num == -1:
        num = len(lemmas)
    if skip_known is not None:
        lemmas = [lemma for lemma in lemmas if lemma['lemma'] not in skip_known]
    if only_known is not None:
        lemmas = [lemma for lemma in lemmas if lemma['lemma'] in only_known]
    if min_length > 1:
        lemmas = [lemma for lemma in lemmas if len(lemma['lemma']) >= min_length]
    lemmas = sorted(lemmas, key=lambda x: x[sort], reverse=reverse)
    return lemmas[:num]

get_words_by_pos(pos, strict=True)

Return words matching a part-of-speech code.

Source code in hsg/classes/subtlexch.py 
134
135
136
137
138
def get_words_by_pos(self, pos: str, strict: bool = True) -> list[dict[str, Any]]:
    """Return words matching a part-of-speech code."""
    if strict:
        return [w for w in self.word_pos_freq if w['dominant_pos'] == pos]
    return [w for w in self.word_pos_freq if pos in [p.lower() for p in w['all_pos'].split('.')]]

load_csv(csvfile, fields)

Parse a SUBTLEX-CH CSV with the given field names.

Source code in hsg/classes/subtlexch.py 
 99
100
101
102
103
104
105
106
107
def load_csv(self, csvfile: str, fields: list[str]) -> list[dict[str, Any]]:
    """Parse a SUBTLEX-CH CSV with the given field names."""
    with open(csvfile) as f:
        reader = csv.DictReader(f, fieldnames=fields, delimiter='\t')
        reader_no_headers = list(reader)[3:]  # skip first 3 lines
        for idx, lemma in enumerate(reader_no_headers):
            lemma['rank'] = idx + 1
            lemma['count_x_cd'] = int(lemma['count']) * int(lemma['cd'])
        return reader_no_headers

Ccedict

hsg.classes.ccedict

Ccedict(cedictfile, frequencies_corpus)

CC-CEDICT dictionary reader with HSK and frequency enrichment.

Loads the dictionary from a cedict_ts.u8 file (or a pickled cache). Supports search by simplified characters, pinyin, or English gloss, with optional HSK-level and frequency-rank filtering/sorting.

Load the CC-CEDICT dictionary and initialise HSK + frequency backends.

Source code in hsg/classes/ccedict.py 
38
39
40
41
42
43
44
45
def __init__(self, cedictfile: str, frequencies_corpus: str) -> None:
    """Load the CC-CEDICT dictionary and initialise HSK + frequency backends."""
    self.cedictfile: str = cedictfile
    self.dictionary: list[dict[str, Any]] = []
    self.fq: Frequency = create_frequency(frequencies_corpus)
    self.hsk: HSK = HSK()
    self.dict_lines: list[str] = []
    self.load_dict()

get_query_type(query)

Classify a query string as 'simplified', 'pinyin', or 'english'.

Source code in hsg/classes/ccedict.py 
 98
 99
100
101
102
103
104
105
106
107
108
109
def get_query_type(self, query: str) -> str:
    """Classify a query string as 'simplified', 'pinyin', or 'english'."""
    pattern_hanzi: re.Pattern[str] = re.compile(
        '^[^' + ADDITIONAL_CHARACTERS.replace('[', r'\[').replace(']', r'\]') + ']*$'
    )
    pattern_pinyin: re.Pattern[str] = re.compile(r'[a-z:]{2,5}\d')
    if pattern_hanzi.match(query):
        return QueryType.SIMPLIFIED.value
    elif pattern_pinyin.match(query):
        return QueryType.PINYIN.value
    else:
        return QueryType.ENGLISH.value

load_dict()

Load the dictionary from pickle cache or parse the raw .u8 file.

Source code in hsg/classes/ccedict.py 
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
def load_dict(self) -> None:
    """Load the dictionary from pickle cache or parse the raw .u8 file."""
    pickle_dict = os.path.join(ASSETS_DIR_PATH, 'ccedict.pickle')
    if os.path.isfile(pickle_dict):
        with open(pickle_dict, 'rb') as d:
            loaded: list[dict[str, Any]] = pickle.load(d)
            self.dictionary = loaded
            logger.info('Loaded ccedict from pickle (%d entries)', len(self.dictionary))
            return
    logger.info('Building ccedict pickle from %s', self.cedictfile)
    with open(self.cedictfile) as f:
        text: str = f.read()
        lines: list[str] = text.split('\n')
        self.dict_lines = list(lines)
        for line in self.dict_lines:
            self.parse_line(line)
        self.remove_surnames()
        with open(pickle_dict, 'wb') as d:
            pickle.dump(self.dictionary, d)

output(words, format)

Print search results in the specified format (csv, json, tabulate).

Source code in hsg/classes/ccedict.py 
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
def output(self, words: list[dict[str, Any]], format: str) -> None:
    """Print search results in the specified format (csv, json, tabulate)."""
    if format == 'csv':
        writer: csv.DictWriter[str] = csv.DictWriter(
            sys.stdout, fieldnames=list(words[0].keys()), delimiter='\t', extrasaction='ignore'
        )
        writer.writeheader()
        writer.writerows(words[:10])
    elif format == 'json':
        print(json.dumps(words))
    elif format == 'tabulate':
        for w in words:
            w.pop('count_x_cd', None)
            if len(str(w['frequency'])) > 5:
                w['frequency'] = ''
            w['english'] = w['english'][:70]
        print(tabulate(words, headers='keys', tablefmt='github'))

parse_line(line)

Parse a single CC-CEDICT line and append to self.dictionary.

Source code in hsg/classes/ccedict.py 
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
def parse_line(self, line: str) -> None:
    """Parse a single CC-CEDICT line and append to self.dictionary."""
    if line == '':
        self.dict_lines.remove(line)
        return
    line_parts = line.rstrip('/').split('/')
    if len(line_parts) <= 1:
        return
    char_and_pinyin: list[str] = line_parts[0].split('[')
    characters: list[str] = char_and_pinyin[0].split()
    frequencies = self.fq.find_word(characters[1])
    hsk_level = self.hsk.get_hsk_new_word_level(characters[1])
    self.dictionary.append(
        {
            'simplified': characters[1],
            'traditional': characters[0],
            'pinyin': char_and_pinyin[1].rstrip().rstrip(']'),
            'english': line_parts[1].replace('|', '/'),
            'hsk': hsk_level if hsk_level else '',
            'frequency': frequencies['rank'] if frequencies else '',
            'count_x_cd': frequencies['count_x_cd'] if frequencies else '',
        }
    )

remove_surnames()

Filter out surname entries from the dictionary.

Source code in hsg/classes/ccedict.py 
91
92
93
94
95
96
def remove_surnames(self) -> None:
    """Filter out surname entries from the dictionary."""
    for x in range(len(self.dictionary) - 1, -1, -1):
        entry = self.dictionary[x]
        if 'surname ' in entry['english'] and entry['traditional'] == self.dictionary[x + 1]['traditional']:
            self.dictionary.pop(x)

search(query, exact, show_traditional, format, sort, reverse, max_hsk, max_results, all_results)

Search the dictionary and print results in the chosen format.

Source code in hsg/classes/ccedict.py 
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
def search(
    self,
    query: str,
    exact: bool,
    show_traditional: bool,
    format: str,
    sort: str,
    reverse: bool,
    max_hsk: str,
    max_results: int,
    all_results: bool,
) -> None:
    """Search the dictionary and print results in the chosen format."""
    search_field: str = self.get_query_type(query)
    query = query.replace(' ', '').lower()

    words: list[dict[str, Any]] = []
    if exact:
        words = [w for w in self.dictionary if w[search_field].replace(' ', '').lower() == query]
    else:
        words = [w for w in self.dictionary if query in w[search_field].replace(' ', '').lower()]

    # filter results by hsk level
    if max_hsk:
        if max_hsk == '7':
            max_hsk = '7-9'
        words = [w for w in words if w['hsk'] and w['hsk'] <= max_hsk]

    # sort results
    words = sorted(words, key=lambda x: self.sort_key(x[sort]), reverse=reverse)

    if not show_traditional:
        for w in words:
            del w['traditional']

    if not all_results and max_results > 0:
        words = words[:max_results]

    self.output(words, format)

sort_key(value)

Return a numeric sort key for HSK level or frequency rank.

Source code in hsg/classes/ccedict.py 
151
152
153
154
155
def sort_key(self, value: str | int) -> int:
    """Return a numeric sort key for HSK level or frequency rank."""
    if value == '7-9':
        return 7
    return int(value) if value else sys.maxsize

QueryType

Bases: Enum

Query classification for CC-CEDICT search: simplified, pinyin, or english.

SentenceCorpus

hsg.classes.sentencecorpus

SentenceCorpus

ABC for sentence corpora (Tatoeba, CC-CEDICT examples, etc.).

find_random_sentences(number, min_length=10, known_chars=None, reverse=False)

Find random sentences of minimum length, optionally filtered.

Source code in hsg/classes/sentencecorpus.py 
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
def find_random_sentences(
    self,
    number: int,
    min_length: int = 10,
    known_chars: set[str] | None = None,
    reverse: bool = False,
) -> list[dict[str, Any]]:
    """Find random sentences of minimum length, optionally filtered."""
    data = self.load()
    candidates: list[dict[str, Any]] = []
    for hanzi, translations in data.items():
        if len(hanzi) < min_length:
            continue
        if known_chars is not None and not all(c in known_chars for c in hanzi):
            continue
        candidates.append({'hanzi': hanzi, 'translations': translations})
    if len(candidates) < number:
        number = len(candidates)
    sampled = _random.sample(candidates, number)
    sampled.sort(key=lambda x: len(x['hanzi']), reverse=reverse)
    return sampled

find_sentences(keyword, known_chars=None, max_sentences=10000, reverse=False)

Find sentences containing keyword, optionally filtered by known set.

Source code in hsg/classes/sentencecorpus.py 
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
def find_sentences(
    self,
    keyword: str,
    known_chars: set[str] | None = None,
    max_sentences: int = 10000,
    reverse: bool = False,
) -> list[dict[str, Any]]:
    """Find sentences containing keyword, optionally filtered by known set."""
    data = self.load()
    sentences: list[dict[str, Any]] = []
    for hanzi, translations in data.items():
        if keyword not in hanzi:
            continue
        if known_chars is not None and not all(c in known_chars for c in hanzi):
            continue
        sentences.append({'hanzi': hanzi, 'translations': translations})
    sentences.sort(key=lambda x: len(x['hanzi']), reverse=reverse)
    return sentences[:max_sentences]

load() abstractmethod

Load sentences into a {hanzi: [translations]} dict.

Source code in hsg/classes/sentencecorpus.py 
 9
10
11
12
@abstractmethod
def load(self) -> dict[str, list[str]]:
    """Load sentences into a {hanzi: [translations]} dict."""
    raise NotImplementedError

TatoebaCorpus

hsg.classes.tatoeba_corpus

TatoebaCorpus(filepath)

Bases: SentenceCorpus

SentenceCorpus backed by Tatoeba TSV.

Source code in hsg/classes/tatoeba_corpus.py 
 9
10
11
def __init__(self, filepath: str) -> None:
    self.filepath = filepath
    self._data: dict[str, list[str]] | None = None

StoryStore

hsg.classes.stories

StoryStore(filepath)

Disk-based story store. Reads stories from a JSON file.

JSON format: {"": {"keyword": str, "keyword_ita": str, "story": str, ...}}

Load stories from the JSON file at filepath.

Source code in hsg/classes/stories.py 
31
32
33
34
35
def __init__(self, filepath: str) -> None:
    """Load stories from the JSON file at filepath."""
    self.filepath = filepath
    self._data: dict[str, dict[str, Any]] = {}
    self._load()

get_story(hanzi)

Return the story entry for hanzi, with HTML tags stripped. None if not found.

Source code in hsg/classes/stories.py 
42
43
44
45
46
47
48
49
50
def get_story(self, hanzi: str) -> dict[str, Any] | None:
    """Return the story entry for hanzi, with HTML tags stripped. None if not found."""
    entry = self._data.get(hanzi)
    if entry is None:
        return None
    story = dict(entry)
    if 'story' in story:
        story['story'] = _strip_tags(story['story'])
    return story

Factories

create_known_set

hsg.classes.knownset_factory

create_known_set(backend, *, max=-1, frequencies_corpus='subtlexch', filepath=None, **kwargs)

Create a KnownSet instance by backend name.

Parameters:

Name Type Description Default
backend str

'heisig', 'hsk', or 'file'.

 required
max int

Frame limit (heisig) or HSK level cap (hsk). -1 = all.

 -1
frequencies_corpus str

Frequency corpus for heisig backend.

 'subtlexch'
filepath str | None

Path to known-characters file (required for 'file').

 None
Source code in hsg/classes/knownset_factory.py 
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
def create_known_set(
    backend: str,
    *,
    max: int = -1,
    frequencies_corpus: str = 'subtlexch',
    filepath: str | None = None,
    **kwargs: Any,
) -> KnownSet:
    """Create a KnownSet instance by backend name.

    Args:
        backend: 'heisig', 'hsk', or 'file'.
        max: Frame limit (heisig) or HSK level cap (hsk). -1 = all.
        frequencies_corpus: Frequency corpus for heisig backend.
        filepath: Path to known-characters file (required for 'file').
    """
    if backend == 'heisig':
        from hsg.classes.heisig import Heisig

        return Heisig(frequencies_corpus, max)

    if backend == 'hsk':
        from hsg.classes.hsk_knownset import HSKKnownSet

        level = max if max > 0 else 6
        return HSKKnownSet(max_level=level)

    if backend == 'file':
        from hsg.classes.file_knownset import FileKnownSet

        if not filepath:
            raise ValueError("backend 'file' requires filepath")
        return FileKnownSet(filepath)

    raise ValueError(f'unknown known-set backend: {backend}')

create_frequency

hsg.classes.frequency_factory

create_frequency(corpus)

Create a Frequency instance by corpus name.

Source code in hsg/classes/frequency_factory.py 
 4
 5
 6
 7
 8
 9
10
11
12
13
14
def create_frequency(corpus: str) -> Frequency:
    """Create a Frequency instance by corpus name."""
    if corpus == 'renminwang':
        from hsg.classes.renminwang import RenMinWang

        return RenMinWang()
    if corpus == 'subtlexch':
        from hsg.classes.subtlexch import SubtlexCh

        return SubtlexCh()
    raise ValueError(f'unknown frequency corpus: {corpus}')