From 5dbf6a346b49649f6a62ed0034824b3f927fa9bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8F=AF=E4=B9=90=E5=8A=A0=E5=86=B0?= <11869599+addie66@user.noreply.gitee.com> Date: Wed, 30 Sep 2026 16:09:40 +0800 Subject: [PATCH] =?UTF-8?q?```=20feat(shopro=5Fhuifu):=20=E9=93=B6?= =?UTF-8?q?=E8=A1=8C=E6=94=AF=E8=A1=8C=E6=95=B0=E6=8D=AE=E5=AF=BC=E5=85=A5?= =?UTF-8?q?=E5=8A=9F=E8=83=BD=E5=A2=9E=E5=BC=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 添加增量导入模式，默认跳过已存在的联行号，不清空数据库 - 新增 --update-existing 选项支持覆盖更新已存在记录 - 新增 --check 选项用于校验数据库中的重复联行号 - 改进导入统计信息，区分新增、更新、跳过、重复等状态 - 优化 CSV 文件解析，支持多种字段别名映射 - 添加文件内重复检测和 BOM 头处理 - 更新 README 文档说明新的使用方式和命令选项 ``` --- addons/shopro_huifu/README.md | 7 +- .../shopro_huifu/command/ImportBankBranch.php | 60 ++++++- .../HuifuOnboardBankBranchImporter.php | 170 +++++++++++++++--- 3 files changed, 203 insertions(+), 34 deletions(-) diff --git a/addons/shopro_huifu/README.md b/addons/shopro_huifu/README.md index cea0a51..6482ebd 100644 --- a/addons/shopro_huifu/README.md +++ b/addons/shopro_huifu/README.md @@ -102,8 +102,11 @@ php think shopro_huifu:import-bank-branch --download # https://hf-mirror.com/datasets/gweesin/CNAPS/resolve/main/cnaps.csv php think shopro_huifu:import-bank-branch --file=addons/shopro_huifu/data/cnaps.csv -# 可选：清空后全量重导 -php think shopro_huifu:import-bank-branch --download --truncate +# 增量导入（默认跳过已存在联行号，不清库） +php think shopro_huifu:import-bank-branch --file=addons/shopro_huifu/data/cnaps.csv + +# 校验库内重复 +php think shopro_huifu:import-bank-branch --check # 试跑（不写库） php think shopro_huifu:import-bank-branch --file=addons/shopro_huifu/data/cnaps.csv --limit=100 --dry-run diff --git a/addons/shopro_huifu/command/ImportBankBranch.php b/addons/shopro_huifu/command/ImportBankBranch.php index a7764c2..69b943c 100644 --- a/addons/shopro_huifu/command/ImportBankBranch.php +++ b/addons/shopro_huifu/command/ImportBankBranch.php @@ -19,7 +19,9 @@ class ImportBankBranch extends Command ->addOption('file', 'f', Option::VALUE_OPTIONAL, 'CSV/JSON source file path') ->addOption('download', 'd', Option::VALUE_NONE, 'Download gweesin/CNAPS cnaps.csv then import') ->addOption('seed', null, Option::VALUE_NONE, 'Import bundled data/bank_branches.json seed only') - ->addOption('truncate', 't', Option::VALUE_NONE, 'Clear table before import') + ->addOption('truncate', 't', Option::VALUE_NONE, 'Clear table before import (optional, default skip existing)') + ->addOption('update-existing', 'u', Option::VALUE_NONE, 'Update rows when branch_code already exists') + ->addOption('check', 'c', Option::VALUE_NONE, 'Check duplicate branch_code / branch_name in DB only') ->addOption('limit', 'l', Option::VALUE_OPTIONAL, 'Max rows to import (0=all)', 0) ->addOption('batch', 'b', Option::VALUE_OPTIONAL, 'Batch size', 500) ->addOption('format', null, Option::VALUE_OPTIONAL, 'Force format: csv|json|auto', 'auto') @@ -35,6 +37,10 @@ class ImportBankBranch extends Command return $this->exportJson($output, $exportJson, (int) $input->getOption('export-limit')); } + if ((bool) $input->getOption('check')) { + return $this->checkDuplicates($output); + } + try { Db::execute('SELECT 1 FROM `' . config('database.prefix') . 'shopro_huifu_bank_branch` LIMIT 1'); } catch (\Throwable $e) { @@ -82,6 +88,7 @@ class ImportBankBranch extends Command $options = [ 'truncate' => (bool) $input->getOption('truncate'), + 'update_existing' => (bool) $input->getOption('update-existing'), 'limit' => (int) $input->getOption('limit'), 'batch' => (int) $input->getOption('batch'), 'format' => (string) $input->getOption('format'), @@ -94,6 +101,8 @@ class ImportBankBranch extends Command } if ($options['truncate']) { $output->writeln('[truncate] 导入前清空旧数据'); + } else { + $output->writeln('已存在联行号将自动跳过（可用 --update-existing 覆盖更新）'); } try { @@ -104,12 +113,17 @@ class ImportBankBranch extends Command } $output->writeln(sprintf( - '完成: 新增 %d，更新 %d，跳过 %d，有效 %d', + '完成: 解析 %d 行，新增 %d，更新 %d，跳过已存在 %d，文件内重复 %d，无效 %d', + $stats['parsed'] ?? 0, $stats['inserted'], $stats['updated'], - $stats['skipped'], - $stats['total'] + $stats['skipped_existing'] ?? 0, + $stats['duplicate_in_file'] ?? 0, + $stats['skipped_invalid'] ?? 0 )); + if (($stats['parsed'] ?? 0) > 1000 && ($stats['inserted'] ?? 0) < 1000 && ($stats['skipped_existing'] ?? 0) < 100) { + $output->writeln('警告: 解析行数很多但新增很少，请确认 CSV 联行号列为 BankCode（非 BankId）'); + } $count = Db::name('shopro_huifu_bank_branch')->where('status', 1)->count(); $output->writeln('当前库内有效支行: ' . $count); @@ -117,6 +131,44 @@ class ImportBankBranch extends Command return 0; } + protected function checkDuplicates(Output $output) + { + try { + $result = HuifuOnboardBankBranchImporter::checkDatabaseDuplicates(); + } catch (\Throwable $e) { + $output->error($e->getMessage()); + return 1; + } + + $output->writeln('当前库内支行总数: ' . $result['total']); + $codeDupes = $result['duplicate_codes']; + if ($codeDupes) { + $output->writeln('发现重复联行号 ' . count($codeDupes) . ' 组（不应出现，表有唯一索引）:'); + foreach ($codeDupes as $item) { + $output->writeln(' ' . $item['branch_code'] . ' x' . $item['cnt']); + } + } else { + $output->writeln('联行号 branch_code 无重复'); + } + + $nameDupes = $result['duplicate_names']; + if ($nameDupes) { + $output->writeln('同名不同号（展示前 ' . count($nameDupes) . ' 组，通常可忽略）:'); + foreach ($nameDupes as $item) { + $output->writeln(sprintf( + ' %s（%d 条，%d 个联行号）', + $item['branch_name'], + $item['cnt'], + $item['code_cnt'] + )); + } + } else { + $output->writeln('支行名称无重复'); + } + + return 0; + } + protected function exportJson(Output $output, $dest, $limit) { try { diff --git a/addons/shopro_huifu/library/onboard/HuifuOnboardBankBranchImporter.php b/addons/shopro_huifu/library/onboard/HuifuOnboardBankBranchImporter.php index 4ddd77d..cc1a390 100644 --- a/addons/shopro_huifu/library/onboard/HuifuOnboardBankBranchImporter.php +++ b/addons/shopro_huifu/library/onboard/HuifuOnboardBankBranchImporter.php @@ -8,8 +8,14 @@ class HuifuOnboardBankBranchImporter { protected static $areaNameMap; + /** @var array */ + protected static $existingCodes; + + /** @var array */ + protected static $seenCodes; + /** - * @return array{inserted:int,updated:int,skipped:int,total:int} + * @return array{inserted:int,updated:int,skipped_invalid:int,skipped_existing:int,duplicate_in_file:int,parsed:int,total:int} */ public static function importFile($file, array $options = []) { @@ -27,49 +33,104 @@ class HuifuOnboardBankBranchImporter $batch = max(50, min(2000, (int) ($options['batch'] ?? 500))); $dryRun = !empty($options['dry_run']); $truncate = !empty($options['truncate']); + $updateExisting = !empty($options['update_existing']); if ($truncate && !$dryRun) { Db::name('shopro_huifu_bank_branch')->where('id', '>', 0)->delete(); } + self::$existingCodes = null; + self::$seenCodes = []; + if (!$dryRun) { + self::loadExistingCodes(); + } + $rows = $format === 'json' ? self::parseJsonFile($file, $limit) : self::parseCsvFile($file, $limit); $inserted = 0; $updated = 0; - $skipped = 0; + $skippedInvalid = 0; + $skippedExisting = 0; + $duplicateInFile = 0; $buffer = []; + $flushOptions = [ + 'dry_run' => $dryRun, + 'update_existing' => $updateExisting, + ]; foreach ($rows as $row) { $normalized = self::normalizeRow($row); if (!$normalized) { - $skipped++; + $skippedInvalid++; continue; } + $code = $normalized['branch_code']; + if (isset(self::$seenCodes[$code])) { + $duplicateInFile++; + continue; + } + self::$seenCodes[$code] = true; + $buffer[] = $normalized; if (count($buffer) >= $batch) { - $stats = self::flushBuffer($buffer, $dryRun); + $stats = self::flushBuffer($buffer, $flushOptions); $inserted += $stats['inserted']; $updated += $stats['updated']; + $skippedExisting += $stats['skipped_existing']; $buffer = []; } } if ($buffer) { - $stats = self::flushBuffer($buffer, $dryRun); + $stats = self::flushBuffer($buffer, $flushOptions); $inserted += $stats['inserted']; $updated += $stats['updated']; + $skippedExisting += $stats['skipped_existing']; } return [ 'inserted' => $inserted, 'updated' => $updated, - 'skipped' => $skipped, + 'skipped_invalid' => $skippedInvalid, + 'skipped_existing' => $skippedExisting, + 'duplicate_in_file' => $duplicateInFile, + 'parsed' => count($rows), 'total' => $inserted + $updated, ]; } + /** + * 校验库内重复联行号 / 同名不同号 + * + * @return array{total:int,duplicate_codes:array,duplicate_names:array} + */ + public static function checkDatabaseDuplicates($nameSampleLimit = 10) + { + $nameSampleLimit = max(1, min(50, (int) $nameSampleLimit)); + $total = (int) Db::name('shopro_huifu_bank_branch')->count(); + + $duplicateCodes = Db::name('shopro_huifu_bank_branch') + ->field('branch_code, COUNT(*) AS cnt') + ->group('branch_code') + ->having('cnt > 1') + ->select(); + + $duplicateNames = Db::query( + 'SELECT branch_name, COUNT(*) AS cnt, COUNT(DISTINCT branch_code) AS code_cnt ' + . 'FROM `' . config('database.prefix') . 'shopro_huifu_bank_branch` ' + . 'GROUP BY branch_name HAVING cnt > 1 ' + . 'ORDER BY cnt DESC LIMIT ' . $nameSampleLimit + ); + + return [ + 'total' => $total, + 'duplicate_codes' => $duplicateCodes ?: [], + 'duplicate_names' => $duplicateNames ?: [], + ]; + } + /** * @return array{inserted:int,updated:int,skipped:int,total:int} */ @@ -207,24 +268,35 @@ class HuifuOnboardBankBranchImporter protected static function normalizeHeaderKey($name) { $name = trim((string) $name); + $name = preg_replace('/^\xEF\xBB\xBF/', '', $name); if ($name === '') { return ''; } $lower = strtolower($name); $aliases = [ - 'branch_code' => ['branch_code', 'bankid', 'cnaps', 'cnapscode', '联行号', '行号', '开户行联行号', '大额支付行号'], + // gweesin/CNAPS: 联行号在 BankCode，BankId 是银行机构号（多支行共用，不可用） + 'branch_code' => ['branch_code', 'bankcode', 'cnaps', 'cnapscode', '联行号', '行号', '开户行联行号', '大额支付行号'], 'branch_name' => ['branch_name', 'lname', 'banknamebranch', '支行名称', '开户行', '开户行支行名称', '支行全称', '网点名称'], - 'bank_name' => ['bank_name', 'bankname', 'banknamemain', '银行', '银行名称', '银行简称', '总行名称'], + 'bank_name' => ['bank_name', 'banknamemain', '银行', '银行名称', '银行简称', '总行名称'], 'province' => ['province', 'provincename', 'areaprovince', '省份', '省'], 'city' => ['city', 'cityname', 'areacity', '城市', '市', '地区'], 'prov_id' => ['prov_id', 'provincecode', '省码'], - 'area_id' => ['area_id', 'citycode', '市码'], + 'area_id' => ['area_id', '市码'], ]; foreach ($aliases as $canonical => $list) { if (in_array($lower, $list, true) || in_array($name, $list, true)) { return $canonical; } } + if ($lower === 'bankname') { + return 'bank_name'; + } + if ($lower === 'citycode') { + return 'city_code'; + } + if ($lower === 'bankid') { + return 'bank_inst_id'; + } return ''; } @@ -252,9 +324,7 @@ class HuifuOnboardBankBranchImporter protected static function normalizeRow(array $row) { - $branchCode = self::normalizeBranchCode( - $row['branch_code'] ?? ($row['BankId'] ?? ($row['cnapsCode'] ?? '')) - ); + $branchCode = self::normalizeBranchCode(self::pickBranchCode($row)); $branchName = trim((string) ($row['branch_name'] ?? ($row['LName'] ?? ''))); if ($branchCode === '' || $branchName === '') { return null; @@ -295,6 +365,16 @@ class HuifuOnboardBankBranchImporter ]; } + protected static function pickBranchCode(array $row) + { + foreach (['branch_code', 'BankCode', 'cnapsCode', '联行号'] as $key) { + if (!empty($row[$key])) { + return $row[$key]; + } + } + return ''; + } + protected static function normalizeBranchCode($code) { $code = preg_replace('/\D+/', '', (string) $code); @@ -418,36 +498,70 @@ class HuifuOnboardBankBranchImporter return strtolower($level . '|' . HuifuOnboardAreaHelper::toCode($provId) . '|' . $name); } + protected static function loadExistingCodes() + { + if (self::$existingCodes !== null) { + return; + } + self::$existingCodes = []; + $codes = Db::name('shopro_huifu_bank_branch')->column('branch_code'); + foreach ($codes ?: [] as $code) { + self::$existingCodes[(string) $code] = true; + } + } + /** * @param array> $rows - * @return array{inserted:int,updated:int} + * @param array{dry_run?:bool,update_existing?:bool} $options + * @return array{inserted:int,updated:int,skipped_existing:int} */ - protected static function flushBuffer(array $rows, $dryRun = false) + protected static function flushBuffer(array $rows, array $options = []) { + $dryRun = !empty($options['dry_run']); + $updateExisting = !empty($options['update_existing']); $inserted = 0; $updated = 0; + $skippedExisting = 0; + if ($dryRun) { - return ['inserted' => count($rows), 'updated' => 0]; + foreach ($rows as $row) { + if (isset(self::$existingCodes[$row['branch_code']])) { + $skippedExisting++; + } else { + $inserted++; + } + } + return ['inserted' => $inserted, 'updated' => 0, 'skipped_existing' => $skippedExisting]; } + $toInsert = []; foreach ($rows as $row) { - $exists = Db::name('shopro_huifu_bank_branch')->where('branch_code', $row['branch_code'])->find(); - if ($exists) { - Db::name('shopro_huifu_bank_branch')->where('id', $exists['id'])->update([ - 'branch_name' => $row['branch_name'], - 'bank_name' => $row['bank_name'], - 'prov_id' => $row['prov_id'], - 'area_id' => $row['area_id'], - 'status' => 1, - ]); - $updated++; + $code = $row['branch_code']; + if (isset(self::$existingCodes[$code])) { + if ($updateExisting) { + Db::name('shopro_huifu_bank_branch')->where('branch_code', $code)->update([ + 'branch_name' => $row['branch_name'], + 'bank_name' => $row['bank_name'], + 'prov_id' => $row['prov_id'], + 'area_id' => $row['area_id'], + 'status' => 1, + ]); + $updated++; + } else { + $skippedExisting++; + } continue; } - Db::name('shopro_huifu_bank_branch')->insert($row); - $inserted++; + $toInsert[] = $row; + self::$existingCodes[$code] = true; } - return ['inserted' => $inserted, 'updated' => $updated]; + if ($toInsert) { + Db::name('shopro_huifu_bank_branch')->insertAll($toInsert); + $inserted = count($toInsert); + } + + return ['inserted' => $inserted, 'updated' => $updated, 'skipped_existing' => $skippedExisting]; } protected static function downloadToFile($url, $dest)