362 lines
19 KiB
Python
362 lines
19 KiB
Python
import re
|
||
import uuid
|
||
from collections import OrderedDict
|
||
from datetime import date
|
||
from decimal import Decimal, InvalidOperation, ROUND_HALF_UP
|
||
|
||
CATEGORIES = ['餐饮', '住宿', '交通', '办公用品', '招待', '采购', '其他']
|
||
HAN = r'\u3400-\u9fff'
|
||
DATE = r'(20\d{2})[年./-](1[0-2]|0?[1-9])[月./-](3[01]|[12]\d|0?[1-9])日?'
|
||
AMOUNT_PATTERNS = [
|
||
r'(?:支付总额|订单总额|应付总额|总金额|总计|价税合计|小写|实付金额|支付金额|付款金额|订单金额|合计|金额)[^0-9]{0,10}([0-9]{1,8}(?:[.,][0-9]{1,2})?)',
|
||
r'(?:¥|￥|RMB|CNY)\s*([0-9]{1,8}(?:[.,][0-9]{1,2})?)',
|
||
r'(?<![0-9])([0-9]{1,7}[.,][0-9]{2})(?![0-9%])',
|
||
]
|
||
MERCHANT_LABEL = r'销售方名称|销方名称|收款方|收款单位|商户全称|商户名称|商户|付款给|收款人|对方户名|交易对象'
|
||
KEYWORDS = [
|
||
'餐饮 餐厅 饭店 酒楼 外卖 咖啡 奶茶 茶饮 饮品 火锅 烧烤 快餐 小吃 食品 美团外卖 饿了么 tims 星巴克',
|
||
'酒店 宾馆 旅馆 住宿 房费 客房 民宿 公寓 入住 退房',
|
||
'铁路 火车 12306 车次 机票 航空 航班 登机 滴滴 网约车 出租车 打车 货运 运输 加油站 汽油 柴油 停车 高速 地铁 公交',
|
||
'办公用品 办公 文具 打印 复印 纸张 墨盒 硒鼓 耗材 电脑 键盘 鼠标 显示器 软件 会员订阅 桌椅 文件夹 饮用水 桶装水',
|
||
'招待 接待 宴请 商务宴请 客户用餐 会议餐',
|
||
'采购 商品 材料 设备 配件 家具 日用品 百货 商城 京东 淘宝 天猫 超市 便利店 购物',
|
||
]
|
||
|
||
|
||
def unique(values):
|
||
return list(dict.fromkeys(values))
|
||
|
||
|
||
def money(value):
|
||
return Decimal(str(value).replace(',', '.')).quantize(Decimal('.01'), rounding=ROUND_HALF_UP)
|
||
|
||
|
||
def first(pattern, text):
|
||
match = re.search(pattern, text, re.I)
|
||
return match.group(1).strip() if match else ''
|
||
|
||
|
||
def extract(lines):
|
||
text = '\n'.join(lines)
|
||
amounts = unique(format(money(value), '.2f') for pattern in AMOUNT_PATTERNS
|
||
for value in re.findall(pattern, text, re.I) if money(value) >= 0)
|
||
dates = []
|
||
for match in re.finditer(DATE, text):
|
||
try:
|
||
dates.append(date(*map(int, match.groups())).isoformat())
|
||
except ValueError:
|
||
pass
|
||
merchants = re.findall(r'(?:' + MERCHANT_LABEL + r')[：:]?\s*([^\n]{2,40})', text)
|
||
|
||
def clean(value):
|
||
return re.sub(r'[|：:]+$', '', re.sub(r'^.*?(?:' + MERCHANT_LABEL.replace('|商户|', '|') + r')[：:]?', '', value)).strip()
|
||
|
||
def valid(value):
|
||
return (2 <= len(value) <= 45 and re.search('[' + HAN + 'A-Za-z]', value)
|
||
and not re.search('支付成功|交易成功|电子发票|增值税发票|付款方式|订单详情|账单详情|交易详情|商品说明|开票日期|价税合计', value))
|
||
|
||
for index, line in enumerate(lines):
|
||
line = clean(line)
|
||
if re.search('销售方|销方|收款方|收款单位|商户|付款给|收款人|对方户名|交易对象', line) and index + 1 < len(lines):
|
||
following = clean(lines[index + 1])
|
||
if valid(following):
|
||
merchants.append(following)
|
||
if re.search('公司|餐厅|饭店|酒店|宾馆|旅馆|商店|店|超市|便利店|中心|服务部|经营部|商行|科技|网络|运输|航空|铁路|医院|药房|加油站|停车场|物业|平台', line) and valid(line):
|
||
merchants.append(line)
|
||
return dict(status='success', rawText=text, lines=lines, amounts=amounts,
|
||
dates=unique(dates), merchants=unique(value.strip().rstrip('：:|') for value in merchants),
|
||
orderNumbers=unique(re.findall(r'(?:交易单号|商户单号|订单号|流水号|交易号)[：:]?\s*([A-Za-z0-9-]{8,40})', text)),
|
||
travel=extract_travel(text, unique(dates)), message='')
|
||
|
||
|
||
def extract_travel(text, dates):
|
||
travel = dict(type='', travelerName='', departure='', destination='', departureTime='', transportNumber='')
|
||
train = re.search('火车票|铁路电子客票|铁路|车次|检票口|12306', text)
|
||
flight = re.search('航空|航班|登机牌|电子客票行程单|乘机|起飞|客票号', text)
|
||
if not train and not flight:
|
||
return travel
|
||
travel['type'] = 'flight' if flight and not train else 'train'
|
||
for pattern in [r'(?:旅客姓名|乘车人|乘机人|旅客|姓名)[：:]?\s*([' + HAN + r'·]{2,15})',
|
||
r'(?:\d{6,}\*{2,}\d{2,})\s*([' + HAN + r'·]{2,8})']:
|
||
names = [name for name in re.findall(pattern, text) if len(name) <= 8 and not re.search('税务|日期|发票|铁路|车站|公司|名称|代码|号码', name)]
|
||
if names:
|
||
travel['travelerName'] = names[0]
|
||
break
|
||
travel['departure'] = first(r'(?:出发地|出发站|始发站|起点|起飞机场)[：:]?\s*([' + HAN + r'A-Za-z0-9]{2,24})', text)
|
||
travel['destination'] = first(r'(?:目的地|到达站|终点站|终点|到达机场)[：:]?\s*([' + HAN + r'A-Za-z0-9]{2,24})', text)
|
||
route = re.search(r'([' + HAN + r'A-Za-z]{2,20})(?:站|机场)?\s*(?:→|->|—|至)\s*([' + HAN + r'A-Za-z]{2,20})(?:站|机场)?', text)
|
||
if route:
|
||
for index, field in enumerate(['departure', 'destination'], 1):
|
||
travel[field] = travel[field] or re.sub(r'(站|机场)$', '', route.group(index))
|
||
if train:
|
||
stations = unique(re.findall(r'([' + HAN + r']{2,12})站', text))
|
||
if len(stations) >= 2:
|
||
travel['departure'] = travel['departure'] or stations[0]
|
||
travel['destination'] = travel['destination'] or stations[1]
|
||
clock_pattern = r'([01]?\d|2[0-3]):([0-5]\d)'
|
||
for match in re.finditer(DATE + r'[^\n]{0,24}?' + clock_pattern, text):
|
||
if not re.search('开票日期|发票日期', text[max(0, match.start() - 16):match.start()]):
|
||
year, month, day, hour, minute = map(int, match.groups())
|
||
travel['departureTime'] = f'{year:04}-{month:02}-{day:02} {hour:02}:{minute:02}'
|
||
break
|
||
if not travel['departureTime']:
|
||
departure_date = ''
|
||
for match in re.finditer(DATE, text):
|
||
if not re.search('开票日期|发票日期', text[max(0, match.start() - 16):match.start()]):
|
||
year, month, day = map(int, match.groups())
|
||
departure_date = f'{year:04}-{month:02}-{day:02}'
|
||
break
|
||
clock = re.search(clock_pattern, text)
|
||
travel['departureTime'] = ((departure_date or next(iter(dates), '')) + (' %02d:%02d' % tuple(map(int, clock.groups())) if clock else '')).strip()
|
||
pattern = r'(?<![A-Z0-9])([GDCZTKSLY]\d{1,5})(?![A-Z0-9])' if train else r'(?<![A-Z0-9])([A-Z0-9]{2}\d{3,4})(?![A-Z0-9])'
|
||
travel['transportNumber'] = first(pattern, text)
|
||
return travel
|
||
|
||
|
||
def amounts(item):
|
||
return [money(value) for value in item['ocr']['amounts'] if money(value) > 0] if item['ocr']['status'] == 'success' else []
|
||
|
||
|
||
def primary(item):
|
||
values = item['ocr']['amounts']
|
||
return money(values[0]) if values and item['ocr']['status'] == 'success' else None
|
||
|
||
|
||
def text(item):
|
||
return item['ocr']['rawText']
|
||
|
||
|
||
def common(left, right, field):
|
||
return [value for value in left['ocr'][field] if value in right['ocr'][field]]
|
||
|
||
|
||
def date_distance(invoices, payments):
|
||
distances = [abs((date.fromisoformat(left) - date.fromisoformat(right)).days)
|
||
for invoice in invoices for payment in payments
|
||
for left in invoice['ocr']['dates'] for right in payment['ocr']['dates']]
|
||
return min(distances, default=999999)
|
||
|
||
|
||
def similarity(left, right):
|
||
def normalize(value):
|
||
return re.sub('[^' + HAN + 'a-z0-9]', '', re.sub('有限责任公司|股份有限公司|有限公司|公司|商户|门店|收款方', '', value.lower()))
|
||
def grams(value):
|
||
return {value[index:index + 2] for index in range(len(value) - 1)} if len(value) > 1 else {value}
|
||
left, right = normalize(left), normalize(right)
|
||
if not left or not right:
|
||
return 0
|
||
if left in right or right in left:
|
||
return 1
|
||
return len(grams(left) & grams(right)) / len(grams(left) | grams(right))
|
||
|
||
|
||
def merchant_similarity(invoice, payment):
|
||
return max((similarity(left, right) for left in invoice['ocr']['merchants'] for right in payment['ocr']['merchants']), default=0)
|
||
|
||
|
||
def category(items):
|
||
content = ' '.join(text(item) + ' ' + ' '.join(item['ocr']['merchants']) for item in items).lower()
|
||
scores = [sum(3 if len(word) >= 4 else 2 if len(word) >= 3 else 1 for word in words.split() if word in content) for words in KEYWORDS]
|
||
return CATEGORIES[scores.index(max(scores))] if max(scores) else '其他'
|
||
|
||
|
||
def add_match(state, invoices, payments, score, reasons):
|
||
selected_category = category(invoices + payments)
|
||
for item in invoices + payments:
|
||
item['matched'] = True
|
||
state['matches'].append(dict(id=str(uuid.uuid4()), invoices=invoices, payments=payments,
|
||
category=selected_category, matchType='auto', score=score,
|
||
reasons=reasons + ['自动分类：' + selected_category]))
|
||
|
||
|
||
def combinations(values, target):
|
||
sums = {Decimal('0.00'): [[]]}
|
||
for item, amount in values:
|
||
snapshot = [(subtotal, [list(group) for group in groups]) for subtotal, groups in sums.items()]
|
||
for subtotal, groups in snapshot:
|
||
total = subtotal + amount
|
||
if total > target:
|
||
continue
|
||
destination = sums.setdefault(total, [])
|
||
for group in list(groups):
|
||
candidate = group + [item]
|
||
if candidate not in destination:
|
||
destination.append(candidate)
|
||
del destination[2:]
|
||
return [group for group in sums.get(target, []) if len(group) >= 2][:2]
|
||
|
||
|
||
def railway(item):
|
||
return bool(re.search('12306|铁路|火车|车次|中铁', text(item)))
|
||
|
||
|
||
def total_label(item):
|
||
return bool(re.search('支付总额|订单总额|应付总额|总金额|总计|合计', text(item)))
|
||
|
||
|
||
def travel_key(item):
|
||
travel = item['ocr']['travel']
|
||
return tuple(travel[field].lower() for field in ['type', 'departure', 'destination', 'departureTime', 'transportNumber'])
|
||
|
||
|
||
def auto_match(state):
|
||
invoices, payments = state['invoices'], state['payments']
|
||
groups = OrderedDict()
|
||
for invoice in invoices:
|
||
if invoice['ocr']['travel']['type']:
|
||
groups.setdefault(travel_key(invoice), []).append(invoice)
|
||
for group in groups.values():
|
||
if len(group) < 2 or any(primary(item) is None for item in group):
|
||
continue
|
||
total = sum(primary(item) for item in group)
|
||
evidence = [item for item in payments if not item['matched'] and railway(item) and total in amounts(item)]
|
||
if evidence and any(re.search('支付成功|付款成功|交易成功|支付方式|交易单号|商户单号', text(item)) for item in evidence) and date_distance(group, evidence) <= 14:
|
||
reasons = [f'同一行程多人发票合计一致 ¥{total:.2f}', ' + '.join(str(primary(item)) for item in group)]
|
||
if len(evidence) >= 2:
|
||
reasons.append('付款凭证与订单截图已共同归入本次行程')
|
||
add_match(state, group, evidence, 100 if len(evidence) >= 2 else 95, reasons)
|
||
for payment in payments:
|
||
total = primary(payment)
|
||
if payment['matched'] or total is None:
|
||
continue
|
||
values = [(item, primary(item)) for item in invoices if not item['matched'] and primary(item) is not None and 0 < primary(item) < total]
|
||
options = combinations(values, total)
|
||
if len(options) != 1:
|
||
continue
|
||
group, evidence = options[0], [payment]
|
||
for candidate in payments:
|
||
if candidate is payment or candidate['matched']:
|
||
continue
|
||
same_total = primary(candidate) == total or (total in amounts(candidate) and total_label(candidate))
|
||
related = (common(payment, candidate, 'orderNumbers') or merchant_similarity(payment, candidate) >= .35
|
||
or (railway(payment) and railway(candidate) and (date_distance([payment], [candidate]) <= 7 or total_label(payment) or total_label(candidate))))
|
||
if same_total and related:
|
||
evidence.append(candidate)
|
||
score = 70
|
||
reasons = [f'多张发票合计一致 ¥{total:.2f}', ' + '.join(str(primary(item)) for item in group)]
|
||
for condition, reason in [(len(evidence) >= 2, '订单与付款凭证相互印证'),
|
||
(date_distance(group, evidence) <= 7, '行程与付款日期接近'),
|
||
(all(item['ocr']['travel']['type'] for item in group) and len({tuple(item['ocr']['travel'][field] for field in ['departure', 'destination', 'departureTime', 'transportNumber']) for item in group}) == 1, '车次、路线及行程日期一致')]:
|
||
if condition:
|
||
score += 10
|
||
reasons.append(reason)
|
||
if score >= 90:
|
||
add_match(state, group, evidence, score, reasons)
|
||
candidates = []
|
||
for invoice in invoices:
|
||
for payment in payments:
|
||
shared = common(invoice, payment, 'amounts')
|
||
if not shared or invoice['ocr']['status'] != 'success' or payment['ocr']['status'] != 'success':
|
||
continue
|
||
score, reasons = 55, ['金额一致 ¥' + shared[0]]
|
||
merchant = merchant_similarity(invoice, payment)
|
||
if merchant >= .35:
|
||
score += int(25 * merchant + .5)
|
||
reasons.append(f'商户相似 {int(merchant * 100 + .5)}%')
|
||
distance = date_distance([invoice], [payment])
|
||
if distance <= 7:
|
||
score += 15 if distance <= 3 else 8
|
||
reasons.append('日期一致' if distance == 0 else f'日期相差 {distance} 天')
|
||
if common(invoice, payment, 'orderNumbers'):
|
||
score += 20
|
||
reasons.append('交易单号一致')
|
||
for amount in shared:
|
||
if sum(amount in item['ocr']['amounts'] for item in invoices) == 1 and sum(amount in item['ocr']['amounts'] for item in payments) == 1:
|
||
score += 20
|
||
reasons.append(f'金额 ¥{amount} 在本批材料中唯一')
|
||
break
|
||
if score >= 70:
|
||
candidates.append((invoice, payment, min(100, score), reasons))
|
||
candidates.sort(key=lambda candidate: -candidate[2])
|
||
for candidate in candidates:
|
||
invoice, payment, score, reasons = candidate
|
||
if invoice['matched'] or payment['matched']:
|
||
continue
|
||
second = max((other[2] for other in candidates if other is not candidate and (other[0] is invoice or other[1] is payment)
|
||
and not other[0]['matched'] and not other[1]['matched']), default=0)
|
||
if score >= 90 or score - second >= 8:
|
||
add_match(state, [invoice], [payment], score, reasons)
|
||
for invoice in invoices:
|
||
if invoice['matched']:
|
||
continue
|
||
available = sorted([(item, primary(item)) for item in payments if not item['matched'] and primary(item) is not None and primary(item) > 0], key=lambda entry: entry[1])
|
||
options = []
|
||
for total in amounts(invoice):
|
||
options.extend((total, group) for group in combinations([(item, amount) for item, amount in available if amount < total], total))
|
||
if len(options) > 1:
|
||
break
|
||
if len(options) == 1:
|
||
total, evidence = options[0]
|
||
add_match(state, [invoice], evidence, 95, [f'多笔付款合计一致 ¥{total:.2f}', ' + '.join(str(primary(item)) for item in evidence)])
|
||
return state
|
||
|
||
|
||
def material_date(item, issue=False):
|
||
content = text(item)
|
||
if issue:
|
||
match = re.search(r'(?:申请开票(?:日期|时间)?|开票日期|发票开具日期|开具日期)\s*[：:]?\s*' + DATE, content)
|
||
if match:
|
||
try:
|
||
return date(*map(int, match.groups())).isoformat()
|
||
except ValueError:
|
||
pass
|
||
if item['ocr']['dates']:
|
||
return item['ocr']['dates'][0]
|
||
match = re.search(r'(?:^|\D)(\d{1,2})月(\d{1,2})日', content)
|
||
if match:
|
||
try:
|
||
return date(date.today().year, *map(int, match.groups())).isoformat()
|
||
except ValueError:
|
||
pass
|
||
return '9999-12-31'
|
||
|
||
|
||
def invoice_amount(item):
|
||
content = text(item)
|
||
patterns = [r'价税合计[^\n]{0,35}?(?:小写)?[^0-9]{0,12}([0-9][0-9,]*(?:\.\d{1,2})?)',
|
||
r'(?:小写|票价|发票金额|合计金额|总金额|应付金额)\s*[（(]?[人民币RMB]*[）)]?\s*[：:]?\s*[¥￥]?\s*([0-9][0-9,]*(?:\.\d{1,2})?)']
|
||
for pattern in patterns:
|
||
values = re.findall(pattern, content, re.I)
|
||
if values:
|
||
return Decimal(values[-1].replace(',', ''))
|
||
return max([Decimal(value) for value in item['ocr']['amounts']], default=Decimal(0))
|
||
|
||
|
||
def payment_amount(item):
|
||
for pattern in [r'(?:支付总额|实付金额|实际支付金额|付款金额|实付款|实付价|合计)\s*[：:]?\s*[¥￥]?\s*([0-9][0-9,]*(?:\.\d{1,2})?)',
|
||
r'已优惠[^\n]{0,24}?[¥￥]\s*([0-9][0-9,]*(?:\.\d{1,2})?)']:
|
||
values = re.findall(pattern, text(item), re.I)
|
||
if values:
|
||
return abs(Decimal(values[-1].replace(',', '')))
|
||
return abs(primary(item) or Decimal(0))
|
||
|
||
|
||
def payment_total(items):
|
||
result = Decimal(0)
|
||
for index, item in enumerate(items):
|
||
amount = payment_amount(item)
|
||
duplicate = re.search('支付总额|查看订单支付明细|购票成功|订单详情', text(item)) and any(
|
||
other_index != index and re.search('支付成功|付款成功|交易成功|支付方式|付款方式|交易单号|商户单号|转账成功|收款方', text(other))
|
||
and abs(payment_amount(other) - amount) < Decimal('.005') for other_index, other in enumerate(items))
|
||
if not duplicate:
|
||
result += amount
|
||
return result
|
||
|
||
|
||
def enrich(state):
|
||
from match_explanation import explain_match
|
||
|
||
for item in state['invoices'] + state['payments'] + state['photos']:
|
||
item['displayAmount'] = float(invoice_amount(item) if item['type'] == 'invoice' else payment_amount(item))
|
||
item['sortDate'] = material_date(item)
|
||
item['issueDate'] = material_date(item, True)
|
||
lookup = {item['id']: item for item in state['invoices'] + state['payments']}
|
||
for match in state['matches']:
|
||
match['invoices'] = [lookup[item['id']] for item in match['invoices']]
|
||
match['payments'] = [lookup[item['id']] for item in match['payments']]
|
||
match['paymentTotal'] = float(payment_total(match['payments']))
|
||
match['expenseAmount'] = float(sum(max((Decimal(value) for value in item['ocr']['amounts']), default=Decimal(0)) for item in match['invoices']))
|
||
match['explanation'] = explain_match(match)
|
||
state['directoryPaymentTotal'] = float(payment_total(state['payments']))
|
||
return state
|