diff --git a/addons/shopro_huifu/README.md b/addons/shopro_huifu/README.md index 6482ebd..2706886 100644 --- a/addons/shopro_huifu/README.md +++ b/addons/shopro_huifu/README.md @@ -95,15 +95,15 @@ php think shopro_huifu:verify # 1) 导入内置种子(约 120 条,主要城市 + 贵州/广东,无需下载) php think shopro_huifu:import-bank-branch --seed -# 2) 自动下载全国 CSV 并导入(约 20MB,来源 gweesin/CNAPS) +# 2) 下载 CSV 并导入(默认 main:主要省市 + 四大行/农商行,约5万条) php think shopro_huifu:import-bank-branch --download # 3) 手动下载后导入(下载失败时用) # https://hf-mirror.com/datasets/gweesin/CNAPS/resolve/main/cnaps.csv php think shopro_huifu:import-bank-branch --file=addons/shopro_huifu/data/cnaps.csv -# 增量导入(默认跳过已存在联行号,不清库) -php think shopro_huifu:import-bank-branch --file=addons/shopro_huifu/data/cnaps.csv +# 全国全量约15万(一般不需要) +php think shopro_huifu:import-bank-branch --file=addons/shopro_huifu/data/cnaps.csv --preset=full # 校验库内重复 php think shopro_huifu:import-bank-branch --check diff --git a/addons/shopro_huifu/command/ImportBankBranch.php b/addons/shopro_huifu/command/ImportBankBranch.php index 69b943c..39a7fc2 100644 --- a/addons/shopro_huifu/command/ImportBankBranch.php +++ b/addons/shopro_huifu/command/ImportBankBranch.php @@ -22,6 +22,7 @@ class ImportBankBranch extends Command ->addOption('truncate', 't', Option::VALUE_NONE, 'Clear table before import (optional, default skip existing)') ->addOption('update-existing', 'u', Option::VALUE_NONE, 'Update rows when branch_code already exists') ->addOption('check', 'c', Option::VALUE_NONE, 'Check duplicate branch_code / branch_name in DB only') + ->addOption('preset', 'p', Option::VALUE_OPTIONAL, 'Import scope: main(~50k, default) / full(all)', 'main') ->addOption('limit', 'l', Option::VALUE_OPTIONAL, 'Max rows to import (0=all)', 0) ->addOption('batch', 'b', Option::VALUE_OPTIONAL, 'Batch size', 500) ->addOption('format', null, Option::VALUE_OPTIONAL, 'Force format: csv|json|auto', 'auto') @@ -32,6 +33,8 @@ class ImportBankBranch extends Command protected function execute(Input $input, Output $output) { + @ini_set('memory_limit', '256M'); + $exportJson = trim((string) $input->getOption('export-json')); if ($exportJson !== '') { return $this->exportJson($output, $exportJson, (int) $input->getOption('export-limit')); @@ -59,8 +62,14 @@ class ImportBankBranch extends Command $download = (bool) $input->getOption('download'); $seed = (bool) $input->getOption('seed'); + $preset = trim((string) $input->getOption('preset')); + if ($preset === '') { + $preset = 'main'; + } + if ($seed) { $file = ADDON_PATH . 'shopro_huifu/data/bank_branches.json'; + $preset = 'full'; } elseif ($download) { $output->writeln('正在下载全国联行号 CSV(约 20MB,来源 gweesin/CNAPS)...'); try { @@ -86,9 +95,14 @@ class ImportBankBranch extends Command } } + if ($seed || strtolower(pathinfo($file, PATHINFO_EXTENSION)) === 'json') { + $preset = 'full'; + } + $options = [ 'truncate' => (bool) $input->getOption('truncate'), 'update_existing' => (bool) $input->getOption('update-existing'), + 'preset' => $preset, 'limit' => (int) $input->getOption('limit'), 'batch' => (int) $input->getOption('batch'), 'format' => (string) $input->getOption('format'), @@ -96,6 +110,7 @@ class ImportBankBranch extends Command ]; $output->writeln('导入文件: ' . $file); + $output->writeln('导入范围: ' . ($preset === 'full' ? '全国全量(约15万)' : '主要省市 + 四大行/农商行等(约5万)')); if ($options['dry_run']) { $output->writeln('[dry-run] 仅解析,不写入数据库'); } @@ -113,11 +128,13 @@ class ImportBankBranch extends Command } $output->writeln(sprintf( - '完成: 解析 %d 行,新增 %d,更新 %d,跳过已存在 %d,文件内重复 %d,无效 %d', + '完成: 解析 %d 行,命中 %d 行,新增 %d,更新 %d,跳过已存在 %d,范围外 %d,文件内重复 %d,无效 %d', $stats['parsed'] ?? 0, + $stats['matched'] ?? 0, $stats['inserted'], $stats['updated'], $stats['skipped_existing'] ?? 0, + $stats['skipped_filtered'] ?? 0, $stats['duplicate_in_file'] ?? 0, $stats['skipped_invalid'] ?? 0 )); diff --git a/addons/shopro_huifu/library/onboard/HuifuOnboardBankBranchImportFilter.php b/addons/shopro_huifu/library/onboard/HuifuOnboardBankBranchImportFilter.php new file mode 100644 index 0000000..a9e3059 --- /dev/null +++ b/addons/shopro_huifu/library/onboard/HuifuOnboardBankBranchImportFilter.php @@ -0,0 +1,138 @@ + false, 'name' => 'full']; + } + throw new \InvalidArgumentException('未知 preset: ' . $preset . ',可选 main / full'); + } + + protected static function mainPreset() + { + return [ + 'enabled' => true, + 'name' => 'main', + 'description' => '主要省市 + 四大行/交邮/农商行等,约5万条', + 'bank_keywords' => [ + '工商银行', + '农业银行', + '中国银行', + '建设银行', + '交通银行', + '邮储银行', + '邮政储蓄', + '农村商业银行', + '农村合作银行', + '农村信用', + '农商行', + '信用社', + '村镇银行', + '农信', + ], + 'major_provinces' => [ + '北京', '上海', '天津', '重庆', + '河北', '山西', '辽宁', '吉林', '黑龙江', + '江苏', '浙江', '安徽', '福建', '江西', '山东', '河南', '湖北', '湖南', '广东', '海南', + '四川', '贵州', '云南', '陕西', '甘肃', + '内蒙古', '广西', '宁夏', '新疆', + ], + 'major_cities' => [ + '北京', '上海', '天津', '重庆', + '广州', '深圳', '东莞', '佛山', '珠海', '中山', '惠州', '汕头', '湛江', + '杭州', '宁波', '温州', '嘉兴', '金华', '绍兴', '台州', + '南京', '苏州', '无锡', '常州', '南通', '徐州', + '成都', '绵阳', '德阳', + '武汉', '宜昌', '襄阳', + '长沙', '株洲', '湘潭', + '郑州', '洛阳', + '西安', '咸阳', + '合肥', '芜湖', + '福州', '厦门', '泉州', + '济南', '青岛', '烟台', '潍坊', + '沈阳', '大连', + '哈尔滨', '长春', + '石家庄', '唐山', '保定', + '太原', + '南昌', '赣州', + '昆明', '曲靖', + '贵阳', '遵义', '安顺', '毕节', '铜仁', + '南宁', '桂林', '柳州', + '海口', '三亚', + '兰州', + '乌鲁木齐', '呼和浩特', + '银川', + '拉萨', '西宁', + '韶关', '江门', '肇庆', '清远', '茂名', '梅州', + ], + 'max_matched' => 55000, + ]; + } + + public static function matches(array $row, array $preset) + { + if (empty($preset['enabled'])) { + return true; + } + + if (!self::matchBank($row, $preset['bank_keywords'] ?? [])) { + return false; + } + + return self::matchRegion($row, $preset); + } + + protected static function matchBank(array $row, array $keywords) + { + $text = self::rowText($row, ['branch_name', 'LName', 'bank_name', 'BankName']); + foreach ($keywords as $keyword) { + if ($keyword !== '' && mb_strpos($text, $keyword, 0, 'UTF-8') !== false) { + return true; + } + } + return false; + } + + protected static function matchRegion(array $row, array $preset) + { + $province = self::rowText($row, ['province', 'ProvinceName']); + $city = self::rowText($row, ['city', 'CityName']); + $branchName = self::rowText($row, ['branch_name', 'LName']); + + foreach ($preset['major_provinces'] ?? [] as $name) { + if ($name !== '' && (mb_strpos($province, $name, 0, 'UTF-8') !== false || mb_strpos($branchName, $name, 0, 'UTF-8') !== false)) { + return true; + } + } + + foreach ($preset['major_cities'] ?? [] as $name) { + if ($name !== '' && (mb_strpos($city, $name, 0, 'UTF-8') !== false || mb_strpos($branchName, $name, 0, 'UTF-8') !== false)) { + return true; + } + } + + return false; + } + + protected static function rowText(array $row, array $keys) + { + $parts = []; + foreach ($keys as $key) { + if (!empty($row[$key])) { + $parts[] = trim((string) $row[$key]); + } + } + return implode(' ', $parts); + } +} diff --git a/addons/shopro_huifu/library/onboard/HuifuOnboardBankBranchImporter.php b/addons/shopro_huifu/library/onboard/HuifuOnboardBankBranchImporter.php index cc1a390..90790dd 100644 --- a/addons/shopro_huifu/library/onboard/HuifuOnboardBankBranchImporter.php +++ b/addons/shopro_huifu/library/onboard/HuifuOnboardBankBranchImporter.php @@ -8,9 +8,6 @@ class HuifuOnboardBankBranchImporter { protected static $areaNameMap; - /** @var array */ - protected static $existingCodes; - /** @var array */ protected static $seenCodes; @@ -34,73 +31,195 @@ class HuifuOnboardBankBranchImporter $dryRun = !empty($options['dry_run']); $truncate = !empty($options['truncate']); $updateExisting = !empty($options['update_existing']); + $preset = HuifuOnboardBankBranchImportFilter::resolvePreset($options['preset'] ?? 'main'); if ($truncate && !$dryRun) { Db::name('shopro_huifu_bank_branch')->where('id', '>', 0)->delete(); } - self::$existingCodes = null; self::$seenCodes = []; - if (!$dryRun) { - self::loadExistingCodes(); + + if ($format === 'json') { + $rows = self::parseJsonFile($file, $limit); + $stats = self::emptyImportStats(); + $stats['parsed'] = count($rows); + return self::importRowIterator($rows, $stats, [ + 'batch' => $batch, + 'dry_run' => $dryRun, + 'update_existing' => $updateExisting, + 'preset' => $preset, + ]); } - $rows = $format === 'json' - ? self::parseJsonFile($file, $limit) - : self::parseCsvFile($file, $limit); - - $inserted = 0; - $updated = 0; - $skippedInvalid = 0; - $skippedExisting = 0; - $duplicateInFile = 0; - $buffer = []; - $flushOptions = [ + return self::importCsvStream($file, [ + 'limit' => $limit, + 'batch' => $batch, 'dry_run' => $dryRun, 'update_existing' => $updateExisting, - ]; + 'preset' => $preset, + ]); + } + + /** + * @param array{limit:int,batch:int,dry_run:bool,update_existing:bool} $options + */ + protected static function importCsvStream($file, array $options) + { + $handle = fopen($file, 'rb'); + if (!$handle) { + throw new \RuntimeException('无法读取 CSV: ' . $file); + } + + $limit = max(0, (int) ($options['limit'] ?? 0)); + $header = null; + $stats = self::emptyImportStats(); + $buffer = []; + $flushOptions = [ + 'dry_run' => !empty($options['dry_run']), + 'update_existing' => !empty($options['update_existing']), + ]; + $batch = max(50, min(2000, (int) ($options['batch'] ?? 500))); + $preset = $options['preset'] ?? HuifuOnboardBankBranchImportFilter::resolvePreset('main'); + $matched = 0; + $maxMatched = (int) ($preset['max_matched'] ?? 0); + + while (($line = fgetcsv($handle)) !== false) { + if (!$line || self::isEmptyCsvLine($line)) { + continue; + } + if ($header === null) { + $header = self::normalizeHeader($line); + continue; + } + + $stats['parsed']++; + $row = self::mapCsvLine($header, $line); + if (!$row) { + continue; + } + + if (!HuifuOnboardBankBranchImportFilter::matches($row, $preset)) { + $stats['skipped_filtered']++; + if ($limit > 0 && $stats['parsed'] >= $limit) { + break; + } + continue; + } + + $matched++; + if ($maxMatched > 0 && $matched > $maxMatched) { + break; + } - foreach ($rows as $row) { $normalized = self::normalizeRow($row); if (!$normalized) { - $skippedInvalid++; - continue; + $stats['skipped_invalid']++; + } elseif (isset(self::$seenCodes[$normalized['branch_code']])) { + $stats['duplicate_in_file']++; + } else { + self::$seenCodes[$normalized['branch_code']] = true; + $buffer[] = $normalized; } - $code = $normalized['branch_code']; - if (isset(self::$seenCodes[$code])) { - $duplicateInFile++; - continue; - } - self::$seenCodes[$code] = true; - $buffer[] = $normalized; if (count($buffer) >= $batch) { - $stats = self::flushBuffer($buffer, $flushOptions); - $inserted += $stats['inserted']; - $updated += $stats['updated']; - $skippedExisting += $stats['skipped_existing']; + self::mergeImportStats($stats, self::flushBuffer($buffer, $flushOptions)); + $buffer = []; + } + + if ($limit > 0 && $stats['parsed'] >= $limit) { + break; + } + } + fclose($handle); + + if ($buffer) { + self::mergeImportStats($stats, self::flushBuffer($buffer, $flushOptions)); + } + + $stats['matched'] = $matched; + $stats['preset'] = $preset['name'] ?? 'main'; + $stats['total'] = $stats['inserted'] + $stats['updated']; + return $stats; + } + + /** + * @param array> $rows + * @param array{batch:int,dry_run:bool,update_existing:bool,preset?:array} $options + */ + protected static function importRowIterator(array $rows, array $stats, array $options) + { + $buffer = []; + $batch = max(50, min(2000, (int) ($options['batch'] ?? 500))); + $flushOptions = [ + 'dry_run' => !empty($options['dry_run']), + 'update_existing' => !empty($options['update_existing']), + ]; + + $preset = $options['preset'] ?? HuifuOnboardBankBranchImportFilter::resolvePreset('main'); + $matched = 0; + $maxMatched = (int) ($preset['max_matched'] ?? 0); + + foreach ($rows as $row) { + if (!HuifuOnboardBankBranchImportFilter::matches($row, $preset)) { + $stats['skipped_filtered']++; + continue; + } + $matched++; + if ($maxMatched > 0 && $matched > $maxMatched) { + break; + } + + $normalized = self::normalizeRow($row); + if (!$normalized) { + $stats['skipped_invalid']++; + continue; + } + if (isset(self::$seenCodes[$normalized['branch_code']])) { + $stats['duplicate_in_file']++; + continue; + } + self::$seenCodes[$normalized['branch_code']] = true; + $buffer[] = $normalized; + + if (count($buffer) >= $batch) { + self::mergeImportStats($stats, self::flushBuffer($buffer, $flushOptions)); $buffer = []; } } if ($buffer) { - $stats = self::flushBuffer($buffer, $flushOptions); - $inserted += $stats['inserted']; - $updated += $stats['updated']; - $skippedExisting += $stats['skipped_existing']; + self::mergeImportStats($stats, self::flushBuffer($buffer, $flushOptions)); } + $stats['matched'] = $matched; + $stats['preset'] = $preset['name'] ?? 'main'; + $stats['total'] = $stats['inserted'] + $stats['updated']; + return $stats; + } + + protected static function emptyImportStats() + { return [ - 'inserted' => $inserted, - 'updated' => $updated, - 'skipped_invalid' => $skippedInvalid, - 'skipped_existing' => $skippedExisting, - 'duplicate_in_file' => $duplicateInFile, - 'parsed' => count($rows), - 'total' => $inserted + $updated, + 'inserted' => 0, + 'updated' => 0, + 'skipped_invalid' => 0, + 'skipped_existing' => 0, + 'skipped_filtered' => 0, + 'duplicate_in_file' => 0, + 'matched' => 0, + 'parsed' => 0, + 'preset' => 'main', + 'total' => 0, ]; } + protected static function mergeImportStats(array &$base, array $chunk) + { + foreach (['inserted', 'updated', 'skipped_existing'] as $key) { + $base[$key] += (int) ($chunk[$key] ?? 0); + } + } + /** * 校验库内重复联行号 / 同名不同号 * @@ -224,35 +343,6 @@ class HuifuOnboardBankBranchImporter return $decoded; } - protected static function parseCsvFile($file, $limit = 0) - { - $handle = fopen($file, 'rb'); - if (!$handle) { - throw new \RuntimeException('无法读取 CSV: ' . $file); - } - - $header = null; - $rows = []; - while (($line = fgetcsv($handle)) !== false) { - if (!$line || self::isEmptyCsvLine($line)) { - continue; - } - if ($header === null) { - $header = self::normalizeHeader($line); - continue; - } - $assoc = self::mapCsvLine($header, $line); - if ($assoc) { - $rows[] = $assoc; - } - if ($limit > 0 && count($rows) >= $limit) { - break; - } - } - fclose($handle); - return $rows; - } - protected static function normalizeHeader(array $header) { $map = []; @@ -498,18 +588,6 @@ class HuifuOnboardBankBranchImporter return strtolower($level . '|' . HuifuOnboardAreaHelper::toCode($provId) . '|' . $name); } - protected static function loadExistingCodes() - { - if (self::$existingCodes !== null) { - return; - } - self::$existingCodes = []; - $codes = Db::name('shopro_huifu_bank_branch')->column('branch_code'); - foreach ($codes ?: [] as $code) { - self::$existingCodes[(string) $code] = true; - } - } - /** * @param array> $rows * @param array{dry_run?:bool,update_existing?:bool} $options @@ -517,28 +595,23 @@ class HuifuOnboardBankBranchImporter */ protected static function flushBuffer(array $rows, array $options = []) { + if (!$rows) { + return ['inserted' => 0, 'updated' => 0, 'skipped_existing' => 0]; + } + $dryRun = !empty($options['dry_run']); $updateExisting = !empty($options['update_existing']); $inserted = 0; $updated = 0; $skippedExisting = 0; - - if ($dryRun) { - foreach ($rows as $row) { - if (isset(self::$existingCodes[$row['branch_code']])) { - $skippedExisting++; - } else { - $inserted++; - } - } - return ['inserted' => $inserted, 'updated' => 0, 'skipped_existing' => $skippedExisting]; - } + $codes = array_column($rows, 'branch_code'); + $existingSet = self::lookupExistingCodes($codes); $toInsert = []; foreach ($rows as $row) { - $code = $row['branch_code']; - if (isset(self::$existingCodes[$code])) { - if ($updateExisting) { + $code = (string) $row['branch_code']; + if (isset($existingSet[$code])) { + if ($updateExisting && !$dryRun) { Db::name('shopro_huifu_bank_branch')->where('branch_code', $code)->update([ 'branch_name' => $row['branch_name'], 'bank_name' => $row['bank_name'], @@ -552,8 +625,12 @@ class HuifuOnboardBankBranchImporter } continue; } + if ($dryRun) { + $inserted++; + continue; + } $toInsert[] = $row; - self::$existingCodes[$code] = true; + $existingSet[$code] = true; } if ($toInsert) { @@ -564,6 +641,24 @@ class HuifuOnboardBankBranchImporter return ['inserted' => $inserted, 'updated' => $updated, 'skipped_existing' => $skippedExisting]; } + /** + * @param array $codes + * @return array + */ + protected static function lookupExistingCodes(array $codes) + { + $codes = array_values(array_unique(array_filter(array_map('strval', $codes)))); + if (!$codes) { + return []; + } + $existing = Db::name('shopro_huifu_bank_branch')->where('branch_code', 'in', $codes)->column('branch_code'); + $set = []; + foreach ($existing ?: [] as $code) { + $set[(string) $code] = true; + } + return $set; + } + protected static function downloadToFile($url, $dest) { if (function_exists('curl_init')) {