import solver
from PIL import Imagefrom kivy.app import App
from kivy.uix.boxlayout import BoxLayout
from kivy.uix.button import Button
from kivy.uix.label import Label
from kivy.uix.widget import Widget
from kivy.graphics import Color, Lineclass PaintWidget(Widget):color = (254, 0, 0, 1) # Pen color 畫筆顏色thick = 8 # Pen thickness 畫筆粗度def __init__(self, root, **kwargs):super().__init__(**kwargs)self.parent_widget = root# Touch down motion:# If the touch position is located in the painting board, draw lines.# 按下動作：# 如果觸摸位置在畫板內，則在畫板上劃線def on_touch_down(self, touch):with self.canvas:Color(*self.color, mode='rgba')if touch.x > self.width or touch.y < self.parent_widget.height - self.height:returntouch.ud['line'] = Line(points=(touch.x, touch.y), width=self.thick)# Touch move motion:# Draw line with mouse/hand moving# 移動動作：# 隨著鼠標/手指的移動畫線def on_touch_move(self, touch):with self.canvas:if touch.x > self.width or touch.y < self.parent_widget.height - self.height:returntouch.ud['line'].points += [touch.x, touch.y]# Touch up motion:# When ending drawing line, save the picture, and call the prediction component to do prediction# 抬起動作：# 結束畫線，保存圖片成文件，并調用預測相關的組件做預測def on_touch_up(self, touch):if touch.x > self.width or touch.y < self.parent_widget.height - self.height:return#self.parent.parent.do_predictions()def export_image(self):input_img_name = './input_expression.png'self.export_to_png(input_img_name)im = Image.open(input_img_name)x, y = im.sizep = Image.new('RGBA', im.size, (255, 255, 255))p.paste(im, (0, 0, x, y), im)p.save('white_bg.png')return 'white_bg.png'class Recognizer(BoxLayout):def __init__(self, **kwargs):super().__init__(**kwargs)self.number = -1 # Variable to store the predicted number 保存識別的數字的變量self.orientation = 'horizontal' # UI related UI相關self.draw_window()# function to declare the components of the application, and add them to the window# 聲明程序UI組件的函數，并且將它們添加到窗口上def draw_window(self):# Clear button 清除按鈕self.clear_button = Button(text='CLEAR', font_name=HandwrittenMathCalculator.font_name, size_hint=(1, 4 / 45),background_color=(255, 165 / 255, 0, 1))self.solve_button = Button(text='SOLVE', font_name=HandwrittenMathCalculator.font_name, size_hint=(1, 4 / 45),background_color=(255, 165 / 255, 0, 1))# Painting board 畫板self.painter = PaintWidget(self, size_hint=(1, 8 / 9))# Label for hint text 提示文字標簽self.hint_label = Label(font_name=HandwrittenMathCalculator.font_name, size_hint=(1, 1 / 45))# Label for predicted number 識別數字展示標簽self.result_label = Label(font_size=120, size_hint=(1, 1 / 3))# Label for some info 展示一些信息的標簽self.info_board = Label(font_size=24, size_hint=(1, 22 / 45))# BoxLayout 盒子布局first_column = BoxLayout(orientation='vertical', size_hint=(2 / 3, 1))second_column = BoxLayout(orientation='vertical', size_hint=(1 / 3, 1))# Add widgets to the window 將各個組件加到應用窗口上first_column.add_widget(self.painter)first_column.add_widget(self.hint_label)second_column.add_widget(self.result_label)second_column.add_widget(self.info_board)second_column.add_widget(self.solve_button)second_column.add_widget(self.clear_button)self.add_widget(first_column)self.add_widget(second_column)# motion binding 動作綁定# Bind the click of the clear button to the clear_paint function# 將清除按鈕的點擊事件綁定到clear_paint函數上self.clear_button.bind(on_release=self.clear_paint)self.solve_button.bind(on_release=self.solve_expression)self.clear_paint() # Initialize the state of the app 初始化應用狀態(tài)# Clear the painting board and initialize the state of the app.def clear_paint(self):self.painter.export_image()#call solver to solve# Clear the painting board and initialize the state of the app.def clear_paint(self, obj=None):self.painter.canvas.clear()self.number = -1self.result_label.text = '?'self.hint_label.text = 'Write math expression above'self.info_board.text = 'Detected expression:\n'# Extract info from the predictions, and display them on the window# 從預測結果中提取信息，并展示在窗口上def show_info(self, result, detected_expression='8+7'):if result == None:self.number = 'Error'else:self.number = resultself.result_label.text = str(self.number)self.hint_label.text = 'Detected expression and result is shown.Press clear to Retry!'self.info_board.text += detected_expressiondef solve_expression(self, obj=None):img = self.painter.export_image()self.info_board.text = 'Detected expression:\n'(result,detected_expression) = solver.solve(img)self.show_info(result,detected_expression)# Main app class
# 主程序類
class HandwrittenMathCalculator(App):font_name = r'Arial.ttf'def build(self):return Recognizer()
solver
各模塊功能匯總腳本
依次進行圖片二值處理
進行圖像分割
進行非連續(xù)符號合并捆綁
調用CNN分類器進行分類
根據位置進行從左到右從上到下排序，符合算式符號的結合邏輯
至此相當于實現了lexer
調用parser進行語法制導計算，返回結果
def solve(filename,mode = 'product'):original_img, binary_img = read_img_and_convert_to_binary(filename)symbols = binary_img_segment(binary_img, original_img)sort_symbols = sort_characters(symbols)process.detect_uncontinous_symbols(sort_symbols, binary_img)length = len(symbols)symbols_to_be_predicted = normalize_matrix_value([x['src_img'] for x in symbols])predict_input_fn = tf.estimator.inputs.numpy_input_fn(x={"x": np.array(symbols_to_be_predicted)},shuffle=False)predictions = cnn_symbol_classifier.predict(input_fn=predict_input_fn)characters = []for i,p in enumerate(predictions):# print(p['classes'],FILELIST[p['classes']])candidates = get_candidates(p['probabilities'])characters.append({'location':symbols[i]['location'],'candidates':candidates})#print([x['location'] for x in characters])modify_characters(characters)# print('排序后的字符序列')# print([[x['location'], x['candidates']] for x in characters])tokens = process.group_into_tokens(characters)# print('識別出的token')print(tokens)node_list = characters_to_nodes(characters)print(node_list)exp_parser = Exp_parser()result=exp_parser.expression(node_list)str = ''for token in tokens:str += token['token_string']print(result)if result is None:return None, strelse:return (round(result,2),str)
