201 lines
6.1 KiB
Python
201 lines
6.1 KiB
Python
import re
|
||
|
||
from num2words import num2words
|
||
|
||
punctuation = r'[\s,.?!/)\'\]>]'
|
||
alphabet_map = {
|
||
"A": " Ei ",
|
||
"B": " Bee ",
|
||
"C": " See ",
|
||
"D": " Dee ",
|
||
"E": " Eee ",
|
||
"F": " Eff ",
|
||
"G": " Jee ",
|
||
"H": " Eich ",
|
||
"I": " Eye ",
|
||
"J": " Jay ",
|
||
"K": " Kay ",
|
||
"L": " El ",
|
||
"M": " Emm ",
|
||
"N": " Enn ",
|
||
"O": " Ohh ",
|
||
"P": " Pee ",
|
||
"Q": " Queue ",
|
||
"R": " Are ",
|
||
"S": " Ess ",
|
||
"T": " Tee ",
|
||
"U": " You ",
|
||
"V": " Vee ",
|
||
"W": " Double You ",
|
||
"X": " Ex ",
|
||
"Y": " Why ",
|
||
"Z": " Zed " # Zed is weird, as I (da3dsoul) am American, but most of the voice models sound British, so it matches
|
||
}
|
||
|
||
|
||
def preprocess(string):
|
||
# the order for some of these matter
|
||
# For example, you need to remove the commas in numbers before expanding them
|
||
string = remove_surrounded_chars(string)
|
||
string = string.replace('"', '')
|
||
string = string.replace('\u201D', '').replace('\u201C', '') # right and left quote
|
||
string = string.replace('\u201F', '') # italic looking quote
|
||
string = string.replace('\n', ' ')
|
||
string = convert_num_locale(string)
|
||
string = replace_negative(string)
|
||
string = replace_roman(string)
|
||
string = hyphen_range_to(string)
|
||
string = num_to_words(string)
|
||
|
||
# TODO Try to use a ML predictor to expand abbreviations. It's hard, dependent on context, and whether to actually
|
||
# try to say the abbreviation or spell it out as I've done below is not agreed upon
|
||
|
||
# For now, expand abbreviations to pronunciations
|
||
# replace_abbreviations adds a lot of unnecessary whitespace to ensure separation
|
||
string = replace_abbreviations(string)
|
||
string = replace_lowercase_abbreviations(string)
|
||
|
||
# cleanup whitespaces
|
||
# remove whitespace before punctuation
|
||
string = re.sub(rf'\s+({punctuation})', r'\1', string)
|
||
string = string.strip()
|
||
# compact whitespace
|
||
string = ' '.join(string.split())
|
||
|
||
return string
|
||
|
||
|
||
def remove_surrounded_chars(string):
|
||
# first this expression will check if there is a string nested exclusively between a alt=
|
||
# and a style= string. This would correspond to only a the alt text of an embedded image
|
||
# If it matches it will only keep that part as the string, and rend it for further processing
|
||
# Afterwards this expression matches to 'as few symbols as possible (0 upwards) between any
|
||
# asterisks' OR' as few symbols as possible (0 upwards) between an asterisk and the end of the string'
|
||
if re.search(r'(?<=alt=)(.*)(?=style=)', string, re.DOTALL):
|
||
m = re.search(r'(?<=alt=)(.*)(?=style=)', string, re.DOTALL)
|
||
string = m.group(0)
|
||
return re.sub(r'\*[^*]*?(\*|$)', '', string)
|
||
|
||
|
||
def convert_num_locale(text):
|
||
# This detects locale and converts it to American without comma separators
|
||
pattern = re.compile(r'(?:\s|^)\d{1,3}(?:\.\d{3})+(,\d+)(?:\s|$)')
|
||
result = text
|
||
while True:
|
||
match = pattern.search(result)
|
||
if match is None:
|
||
break
|
||
|
||
start = match.start()
|
||
end = match.end()
|
||
result = result[0:start] + result[start:end].replace('.', '').replace(',', '.') + result[end:len(result)]
|
||
|
||
# removes comma separators from existing American numbers
|
||
pattern = re.compile(r'(\d),(\d)')
|
||
result = pattern.sub(r'\1\2', result)
|
||
|
||
return result
|
||
|
||
|
||
def replace_negative(string):
|
||
# handles situations like -5. -5 would become negative 5, which would then be expanded to negative five
|
||
return re.sub(rf'(\s)(-)(\d+)({punctuation})', r'\1negative \3\4', string)
|
||
|
||
|
||
def replace_roman(string):
|
||
# find a string of roman numerals.
|
||
# Only 2 or more, to avoid capturing I and single character abbreviations, like names
|
||
pattern = re.compile(rf'\s[IVXLCDM]{{2,}}{punctuation}')
|
||
result = string
|
||
while True:
|
||
match = pattern.search(result)
|
||
if match is None:
|
||
break
|
||
|
||
start = match.start()
|
||
end = match.end()
|
||
result = result[0:start + 1] + str(roman_to_int(result[start + 1:end - 1])) + result[end - 1:len(result)]
|
||
|
||
return result
|
||
|
||
|
||
def roman_to_int(s):
|
||
rom_val = {'I': 1, 'V': 5, 'X': 10, 'L': 50, 'C': 100, 'D': 500, 'M': 1000}
|
||
int_val = 0
|
||
for i in range(len(s)):
|
||
if i > 0 and rom_val[s[i]] > rom_val[s[i - 1]]:
|
||
int_val += rom_val[s[i]] - 2 * rom_val[s[i - 1]]
|
||
else:
|
||
int_val += rom_val[s[i]]
|
||
return int_val
|
||
|
||
|
||
def hyphen_range_to(text):
|
||
pattern = re.compile(r'(\d+)[-–](\d+)')
|
||
result = pattern.sub(lambda x: x.group(1) + ' to ' + x.group(2), text)
|
||
return result
|
||
|
||
|
||
def num_to_words(text):
|
||
# 1000 or 10.23
|
||
pattern = re.compile(r'\d+\.\d+|\d+')
|
||
result = pattern.sub(lambda x: num2words(float(x.group())), text)
|
||
return result
|
||
|
||
|
||
def replace_abbreviations(string):
|
||
# abbreviations 1 to 4 characters long. It will get things like A and I, but those are pronounced with their letter
|
||
pattern = re.compile(rf'(^|[\s(.\'\[<])([A-Z]{{1,4}})({punctuation}|$)')
|
||
result = string
|
||
while True:
|
||
match = pattern.search(result)
|
||
if match is None:
|
||
break
|
||
|
||
start = match.start()
|
||
end = match.end()
|
||
result = result[0:start] + replace_abbreviation(result[start:end]) + result[end:len(result)]
|
||
|
||
return result
|
||
|
||
|
||
def replace_lowercase_abbreviations(string):
|
||
# abbreviations 1 to 4 characters long, separated by dots i.e. e.g.
|
||
pattern = re.compile(rf'(^|[\s(.\'\[<])(([a-z]\.){{1,4}})({punctuation}|$)')
|
||
result = string
|
||
while True:
|
||
match = pattern.search(result)
|
||
if match is None:
|
||
break
|
||
|
||
start = match.start()
|
||
end = match.end()
|
||
result = result[0:start] + replace_abbreviation(result[start:end].upper()) + result[end:len(result)]
|
||
|
||
return result
|
||
|
||
|
||
def replace_abbreviation(string):
|
||
result = ""
|
||
for char in string:
|
||
result += match_mapping(char)
|
||
|
||
return result
|
||
|
||
|
||
def match_mapping(char):
|
||
for mapping in alphabet_map.keys():
|
||
if char == mapping:
|
||
return alphabet_map[char]
|
||
|
||
return char
|
||
|
||
|
||
def __main__(args):
|
||
print(preprocess(args[1]))
|
||
|
||
|
||
if __name__ == "__main__":
|
||
import sys
|
||
__main__(sys.argv)
|