pendletong/lancaster_stemmer
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 1 Wiki Activity

feat: Initial stemmer feature
Some checks failed
test / test (push) Has been cancelled
Details

Browse Source
This commit is contained in:
Gareth Pendleton
2025-10-31 22:08:14 +00:00
parent 88550985e7
commit 4f470cdf26
8 changed files with 25827 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

491
src/lancaster_stemmer.gleam Normal file
View File

@@ -0,0 +1,491 @@
import gleam/bool
import gleam/dict
import gleam/int
import gleam/io
import gleam/list
import gleam/option.{None, Some}
import gleam/result
import gleam/string
import simplifile
import splitter
pub opaque type Rule {
Rule(
id: String,
letter: String,
suffix: String,
intact: Bool,
remove: Int,
append: String,
restem: Bool,
)
}
pub type Rules =
dict.Dict(String, List(Rule))
const default_rules_list = [
#(
"a",
[
Rule("(1:ai*2.)", "a", "ia", True, 2, "", False),
Rule("(2:a*1.)", "a", "a", True, 1, "", False),
],
),
#("b", [Rule("(3:bb1.)", "b", "bb", False, 1, "", False)]),
#(
"c",
[
Rule("(4:city3s.)", "c", "ytic", False, 3, "s", False),
Rule("(5:ci2>)", "c", "ic", False, 2, "", True),
Rule("(6:cn1t>)", "c", "nc", False, 1, "t", True),
],
),
#(
"d",
[
Rule("(7:dd1.)", "d", "dd", False, 1, "", False),
Rule("(8:dei3y>)", "d", "ied", False, 3, "y", True),
Rule("(9:deec2ss.)", "d", "ceed", False, 2, "ss", False),
Rule("(10:dee1.)", "d", "eed", False, 1, "", False),
Rule("(11:de2>)", "d", "ed", False, 2, "", True),
Rule("(12:dooh4>)", "d", "hood", False, 4, "", True),
],
),
#("e", [Rule("(13:e1>)", "e", "e", False, 1, "", True)]),
#(
"f",
[
Rule("(14:feil1v.)", "f", "lief", False, 1, "v", False),
Rule("(15:fi2>)", "f", "if", False, 2, "", True),
],
),
#(
"g",
[
Rule("(16:gni3>)", "g", "ing", False, 3, "", True),
Rule("(17:gai3y.)", "g", "iag", False, 3, "y", False),
Rule("(18:ga2>)", "g", "ag", False, 2, "", True),
Rule("(19:gg1.)", "g", "gg", False, 1, "", False),
],
),
#(
"h",
[
Rule("(20:ht*2.)", "h", "th", True, 2, "", False),
Rule("(21:hsiug5ct.)", "h", "guish", False, 5, "ct", False),
Rule("(22:hsi3>)", "h", "ish", False, 3, "", True),
],
),
#(
"i",
[
Rule("(23:i*1.)", "i", "i", True, 1, "", False),
Rule("(24:i1y>)", "i", "i", False, 1, "y", True),
],
),
#(
"j",
[
Rule("(25:ji1d.)", "j", "ij", False, 1, "d", False),
Rule("(26:juf1s.)", "j", "fuj", False, 1, "s", False),
Rule("(27:ju1d.)", "j", "uj", False, 1, "d", False),
Rule("(28:jo1d.)", "j", "oj", False, 1, "d", False),
Rule("(29:jeh1r.)", "j", "hej", False, 1, "r", False),
Rule("(30:jrev1t.)", "j", "verj", False, 1, "t", False),
Rule("(31:jsim2t.)", "j", "misj", False, 2, "t", False),
Rule("(32:jn1d.)", "j", "nj", False, 1, "d", False),
Rule("(33:j1s.)", "j", "j", False, 1, "s", False),
],
),
#(
"l",
[
Rule("(34:lbaifi6.)", "l", "ifiabl", False, 6, "", False),
Rule("(35:lbai4y.)", "l", "iabl", False, 4, "y", False),
Rule("(36:lba3>)", "l", "abl", False, 3, "", True),
Rule("(37:lbi3.)", "l", "ibl", False, 3, "", False),
Rule("(38:lib2l>)", "l", "bil", False, 2, "l", True),
Rule("(39:lc1.)", "l", "cl", False, 1, "", False),
Rule("(40:lufi4y.)", "l", "iful", False, 4, "y", False),
Rule("(41:luf3>)", "l", "ful", False, 3, "", True),
Rule("(42:lu2.)", "l", "ul", False, 2, "", False),
Rule("(43:lai3>)", "l", "ial", False, 3, "", True),
Rule("(44:lau3>)", "l", "ual", False, 3, "", True),
Rule("(45:la2>)", "l", "al", False, 2, "", True),
Rule("(46:ll1.)", "l", "ll", False, 1, "", False),
],
),
#(
"m",
[
Rule("(47:mui3.)", "m", "ium", False, 3, "", False),
Rule("(48:mu*2.)", "m", "um", True, 2, "", False),
Rule("(49:msi3>)", "m", "ism", False, 3, "", True),
Rule("(50:mm1.)", "m", "mm", False, 1, "", False),
],
),
#(
"n",
[
Rule("(51:nois4j>)", "n", "sion", False, 4, "j", True),
Rule("(52:noix4ct.)", "n", "xion", False, 4, "ct", False),
Rule("(53:noi3>)", "n", "ion", False, 3, "", True),
Rule("(54:nai3>)", "n", "ian", False, 3, "", True),
Rule("(55:na2>)", "n", "an", False, 2, "", True),
Rule("(56:nee0.)", "n", "een", False, 0, "", False),
Rule("(57:ne2>)", "n", "en", False, 2, "", True),
Rule("(58:nn1.)", "n", "nn", False, 1, "", False),
],
),
#(
"p",
[
Rule("(59:pihs4>)", "p", "ship", False, 4, "", True),
Rule("(60:pp1.)", "p", "pp", False, 1, "", False),
],
),
#(
"r",
[
Rule("(61:re2>)", "r", "er", False, 2, "", True),
Rule("(62:rae0.)", "r", "ear", False, 0, "", False),
Rule("(63:ra2.)", "r", "ar", False, 2, "", False),
Rule("(64:ro2>)", "r", "or", False, 2, "", True),
Rule("(65:ru2>)", "r", "ur", False, 2, "", True),
Rule("(66:rr1.)", "r", "rr", False, 1, "", False),
Rule("(67:rt1>)", "r", "tr", False, 1, "", True),
Rule("(68:rei3y>)", "r", "ier", False, 3, "y", True),
],
),
#(
"s",
[
Rule("(69:sei3y>)", "s", "ies", False, 3, "y", True),
Rule("(70:sis2.)", "s", "sis", False, 2, "", False),
Rule("(71:si2>)", "s", "is", False, 2, "", True),
Rule("(72:ssen4>)", "s", "ness", False, 4, "", True),
Rule("(73:ss0.)", "s", "ss", False, 0, "", False),
Rule("(74:suo3>)", "s", "ous", False, 3, "", True),
Rule("(75:su*2.)", "s", "us", True, 2, "", False),
Rule("(76:s*1>)", "s", "s", True, 1, "", True),
Rule("(77:s0.)", "s", "s", False, 0, "", False),
],
),
#(
"t",
[
Rule("(78:tacilp4y.)", "t", "plicat", False, 4, "y", False),
Rule("(79:ta2>)", "t", "at", False, 2, "", True),
Rule("(80:tnem4>)", "t", "ment", False, 4, "", True),
Rule("(81:tne3>)", "t", "ent", False, 3, "", True),
Rule("(82:tna3>)", "t", "ant", False, 3, "", True),
Rule("(83:tpir2b.)", "t", "ript", False, 2, "b", False),
Rule("(84:tpro2b.)", "t", "orpt", False, 2, "b", False),
Rule("(85:tcud1.)", "t", "duct", False, 1, "", False),
Rule("(86:tpmus2.)", "t", "sumpt", False, 2, "", False),
Rule("(87:tpec2iv.)", "t", "cept", False, 2, "iv", False),
Rule("(88:tulo2v.)", "t", "olut", False, 2, "v", False),
Rule("(89:tsis0.)", "t", "sist", False, 0, "", False),
Rule("(90:tsi3>)", "t", "ist", False, 3, "", True),
Rule("(91:tt1.)", "t", "tt", False, 1, "", False),
],
),
#(
"u",
[
Rule("(92:uqi3.)", "u", "iqu", False, 3, "", False),
Rule("(93:ugo1.)", "u", "ogu", False, 1, "", False),
],
),
#(
"v",
[
Rule("(94:vis3j>)", "v", "siv", False, 3, "j", True),
Rule("(95:vie0.)", "v", "eiv", False, 0, "", False),
Rule("(96:vi2>)", "v", "iv", False, 2, "", True),
],
),
#(
"y",
[
Rule("(97:ylb1>)", "y", "bly", False, 1, "", True),
Rule("(98:yli3y>)", "y", "ily", False, 3, "y", True),
Rule("(99:ylp0.)", "y", "ply", False, 0, "", False),
Rule("(100:yl2>)", "y", "ly", False, 2, "", True),
Rule("(101:ygo1.)", "y", "ogy", False, 1, "", False),
Rule("(102:yhp1.)", "y", "phy", False, 1, "", False),
Rule("(103:ymo1.)", "y", "omy", False, 1, "", False),
Rule("(104:ypo1.)", "y", "opy", False, 1, "", False),
Rule("(105:yti3>)", "y", "ity", False, 3, "", True),
Rule("(106:yte3>)", "y", "ety", False, 3, "", True),
Rule("(107:ytl2.)", "y", "lty", False, 2, "", False),
Rule("(108:yrtsi5.)", "y", "istry", False, 5, "", False),
Rule("(109:yra3>)", "y", "ary", False, 3, "", True),
Rule("(110:yro3>)", "y", "ory", False, 3, "", True),
Rule("(111:yfi3.)", "y", "ify", False, 3, "", False),
Rule("(112:ycn2t>)", "y", "ncy", False, 2, "t", True),
Rule("(113:yca3>)", "y", "acy", False, 3, "", True),
],
),
#(
"z",
[
Rule("(114:zi2>)", "z", "iz", False, 2, "", True),
Rule("(115:zy1s.)", "z", "yz", False, 1, "s", False),
],
),
]
pub fn main() {
io.println("Hello from paicehusk!")
let assert Ok(rules) = load_rules("paice-husk-rules.txt")
stem("abominable", rules)
|> echo
}
pub fn default_rules() -> Rules {
dict.from_list(default_rules_list)
}
pub fn stem(word: String, rules: Rules) -> String {
case is_valid(word) {
True -> {
do_stem(word, rules, True)
}
False -> word
}
}
fn do_stem(word: String, rules: Rules, intact: Bool) -> String {
case string.reverse(word) |> string.pop_grapheme {
Ok(#(letter, _)) -> {
case dict.get(rules, letter) {
Ok(specific_rules) -> {
let #(stem, restem, intact) =
list.fold_until(
specific_rules,
#(word, False, intact),
fn(state, rule) {
case rule_matches(rule, word, intact) {
True -> {
let result = apply_rule(rule, word)
case is_valid(result) {
False -> list.Continue(state)
True -> {
list.Stop(#(result, rule.restem, False))
}
}
}
False -> list.Continue(state)
}
},
)
case restem {
True -> do_stem(stem, rules, intact)
False -> stem
}
}
Error(_) -> word
}
}
Error(_) -> word
}
}
fn rule_matches(rule: Rule, word: String, stem_intact: Bool) -> Bool {
case !stem_intact && rule.intact {
True -> False
False -> string.ends_with(word, rule.suffix)
}
}
fn apply_rule(rule: Rule, word: String) -> String {
string.drop_end(word, rule.remove) <> rule.append
}
fn is_valid(word: String) -> Bool {
case word {
"" -> False
"a" <> rest | "e" <> rest | "i" <> rest | "o" <> rest | "u" <> rest -> {
rest != ""
}
_ -> {
is_valid_internal(word, 0)
}
}
}
fn is_valid_internal(word: String, length: Int) -> Bool {
case word {
"" -> False
"a" <> rest
| "e" <> rest
| "i" <> rest
| "o" <> rest
| "u" <> rest
| "y" <> rest -> {
{ length + 1 + string.length(rest) } >= 3
}
_ -> {
is_valid_internal(string.drop_start(word, 1), length + 1)
}
}
}
pub fn load_rules(filename: String) -> Result(Rules, Nil) {
case simplifile.read(filename) {
Error(_) -> Error(Nil)
Ok(rules) -> {
let split = splitter.new(["\n", "\r\n"])
use rules <- result.try(process_rules(rules, split, [], 1))
list.fold(rules, dict.new(), fn(acc, rule) {
dict.upsert(acc, rule.letter, fn(rules) {
case rules {
Some(rules) -> {
[rule, ..rules]
}
None -> [rule]
}
})
})
|> Ok
}
}
}
fn process_rules(
rules: String,
split: splitter.Splitter,
acc: List(Rule),
id: Int,
) -> Result(List(Rule), Nil) {
case splitter.split(split, rules) {
#("end0." <> _, _, _) -> Ok(acc)
#(rule, _, rest) -> {
use rule <- result.try(process_rule(rule, id))
process_rules(rest, split, [rule, ..acc], id + 1)
}
}
}
fn process_rule(rule: String, id: Int) -> Result(Rule, Nil) {
let rule = string.trim(rule)
use #(suffix, rule) <- result.try(parse_rule_text(rule, ""))
use <- bool.guard(when: suffix == "", return: Error(Nil))
let #(intact, rule) = case rule {
"*" <> rule -> #(True, rule)
_ -> #(False, rule)
}
use #(remove, rule) <- result.try(parse_rule_remove(rule, ""))
use #(append, rule) <- result.try(parse_rule_text(rule, ""))
use restem <- result.try(case rule {
">" <> _ -> Ok(True)
"." <> _ -> Ok(False)
_ -> Error(Nil)
})
let id =
"("
<> int.to_string(id)
<> ":"
<> suffix
<> {
case intact {
True -> "*"
False -> ""
}
}
<> int.to_string(remove)
<> append
<> {
case restem {
True -> ">"
False -> "."
}
}
<> ")"
use letter <- result.try(string.first(suffix))
let suffix = string.reverse(suffix)
Ok(Rule(id, letter, suffix, intact, remove, append, restem))
}
fn parse_rule_remove(rule: String, acc: String) -> Result(#(Int, String), Nil) {
case rule {
"0" as number <> rest
| "1" as number <> rest
| "2" as number <> rest
| "3" as number <> rest
| "4" as number <> rest
| "5" as number <> rest
| "6" as number <> rest
| "7" as number <> rest
| "8" as number <> rest
| "9" as number <> rest -> parse_rule_remove(rest, acc <> number)
_ if acc == "" -> Error(Nil)
_ -> {
use i <- result.try(int.parse(acc))
Ok(#(i, rule))
}
}
}
fn parse_rule_text(rule: String, acc: String) -> Result(#(String, String), Nil) {
case rule {
"a" as letter <> rest
| "b" as letter <> rest
| "c" as letter <> rest
| "d" as letter <> rest
| "e" as letter <> rest
| "f" as letter <> rest
| "g" as letter <> rest
| "h" as letter <> rest
| "i" as letter <> rest
| "j" as letter <> rest
| "k" as letter <> rest
| "l" as letter <> rest
| "m" as letter <> rest
| "n" as letter <> rest
| "o" as letter <> rest
| "p" as letter <> rest
| "q" as letter <> rest
| "r" as letter <> rest
| "s" as letter <> rest
| "t" as letter <> rest
| "u" as letter <> rest
| "v" as letter <> rest
| "w" as letter <> rest
| "x" as letter <> rest
| "y" as letter <> rest
| "z" as letter <> rest
| "A" as letter <> rest
| "B" as letter <> rest
| "C" as letter <> rest
| "D" as letter <> rest
| "E" as letter <> rest
| "F" as letter <> rest
| "G" as letter <> rest
| "H" as letter <> rest
| "I" as letter <> rest
| "J" as letter <> rest
| "K" as letter <> rest
| "L" as letter <> rest
| "M" as letter <> rest
| "N" as letter <> rest
| "O" as letter <> rest
| "P" as letter <> rest
| "Q" as letter <> rest
| "R" as letter <> rest
| "S" as letter <> rest
| "T" as letter <> rest
| "U" as letter <> rest
| "V" as letter <> rest
| "W" as letter <> rest
| "X" as letter <> rest
| "Y" as letter <> rest
| "Z" as letter <> rest -> parse_rule_text(rest, acc <> letter)
_ -> Ok(#(acc, rule))
}
}