wikiboss/xionggeshop
1
0
派生 0
代码 工单 合并请求 Actions 软件包 项目 版本发布 百科 动态

```

浏览代码 
feat(shopro_huifu): 添加联行号导入范围筛选功能

- 新增 preset 参数支持 main(默认) 和 full 两种模式
- main 模式仅导入主要省市 + 四大行/农商行等约5万条数据
- full 模式导入全国全量约15万条数据
- 添加 HuifuOnboardBankBranchImportFilter 类处理筛选逻辑
- 优化 CSV 流式处理，提升大文件导入性能
- 更新 README 文档说明新的导入命令选项
```
这个提交包含在：
可乐加冰
2026-09-30 16:15:04 +08:00
父节点 5dbf6a346b
当前提交 29de6f63b7
共有 4 个文件被更改，包括 353 次插入 和 103 次删除
下载 Patch 文件 下载 Diff 文件 展开所有文件 折叠所有文件

6
addons/shopro_huifu/README.md
查看文件

@@ -95,15 +95,15 @@ php think shopro_huifu:verify
# 1) 导入内置种子（约 120 条，主要城市 + 贵州/广东，无需下载）
php think shopro_huifu:import-bank-branch --seed
# 2) 自动下载全国 CSV 并导入（约 20MB，来源 gweesin/CNAPS）
# 2) 下载 CSV 并导入（默认 main：主要省市 + 四大行/农商行，约5万条）
php think shopro_huifu:import-bank-branch --download
# 3) 手动下载后导入（下载失败时用）
# https://hf-mirror.com/datasets/gweesin/CNAPS/resolve/main/cnaps.csv
php think shopro_huifu:import-bank-branch --file=addons/shopro_huifu/data/cnaps.csv
# 增量导入（默认跳过已存在联行号，不清库）
php think shopro_huifu:import-bank-branch --file=addons/shopro_huifu/data/cnaps.csv
# 全国全量约15万（一般不需要）
php think shopro_huifu:import-bank-branch --file=addons/shopro_huifu/data/cnaps.csv --preset=full
# 校验库内重复
php think shopro_huifu:import-bank-branch --check

19
addons/shopro_huifu/command/ImportBankBranch.php
查看文件

@@ -22,6 +22,7 @@ class ImportBankBranch extends Command
->addOption('truncate', 't', Option::VALUE_NONE, 'Clear table before import (optional, default skip existing)')
->addOption('update-existing', 'u', Option::VALUE_NONE, 'Update rows when branch_code already exists')
->addOption('check', 'c', Option::VALUE_NONE, 'Check duplicate branch_code / branch_name in DB only')
->addOption('preset', 'p', Option::VALUE_OPTIONAL, 'Import scope: main(~50k, default) / full(all)', 'main')
->addOption('limit', 'l', Option::VALUE_OPTIONAL, 'Max rows to import (0=all)', 0)
->addOption('batch', 'b', Option::VALUE_OPTIONAL, 'Batch size', 500)
->addOption('format', null, Option::VALUE_OPTIONAL, 'Force format: csv|json|auto', 'auto')
@@ -32,6 +33,8 @@ class ImportBankBranch extends Command
protected function execute(Input $input, Output $output)
{
@ini_set('memory_limit', '256M');
$exportJson = trim((string) $input->getOption('export-json'));
if ($exportJson !== '') {
return $this->exportJson($output, $exportJson, (int) $input->getOption('export-limit'));
@@ -59,8 +62,14 @@ class ImportBankBranch extends Command
$download = (bool) $input->getOption('download');
$seed = (bool) $input->getOption('seed');
$preset = trim((string) $input->getOption('preset'));
if ($preset === '') {
$preset = 'main';
}
if ($seed) {
$file = ADDON_PATH . 'shopro_huifu/data/bank_branches.json';
$preset = 'full';
} elseif ($download) {
$output->writeln('正在下载全国联行号 CSV（约 20MB，来源 gweesin/CNAPS）...');
try {
@@ -86,9 +95,14 @@ class ImportBankBranch extends Command
}
}
if ($seed || strtolower(pathinfo($file, PATHINFO_EXTENSION)) === 'json') {
$preset = 'full';
}
$options = [
'truncate' => (bool) $input->getOption('truncate'),
'update_existing' => (bool) $input->getOption('update-existing'),
'preset' => $preset,
'limit' => (int) $input->getOption('limit'),
'batch' => (int) $input->getOption('batch'),
'format' => (string) $input->getOption('format'),
@@ -96,6 +110,7 @@ class ImportBankBranch extends Command
];
$output->writeln('导入文件: ' . $file);
$output->writeln('导入范围: ' . ($preset === 'full' ? '全国全量（约15万）' : '主要省市 + 四大行/农商行等（约5万）'));
if ($options['dry_run']) {
$output->writeln('[dry-run] 仅解析，不写入数据库');
}
@@ -113,11 +128,13 @@ class ImportBankBranch extends Command
}
$output->writeln(sprintf(
'完成: 解析 %d 行，新增 %d，更新 %d，跳过已存在 %d，文件内重复 %d，无效 %d',
'完成: 解析 %d 行，命中 %d 行，新增 %d，更新 %d，跳过已存在 %d，范围外 %d，文件内重复 %d，无效 %d',
$stats['parsed'] ?? 0,
$stats['matched'] ?? 0,
$stats['inserted'],
$stats['updated'],
$stats['skipped_existing'] ?? 0,
$stats['skipped_filtered'] ?? 0,
$stats['duplicate_in_file'] ?? 0,
$stats['skipped_invalid'] ?? 0
));

138
addons/shopro_huifu/library/onboard/HuifuOnboardBankBranchImportFilter.php 普通文件
查看文件

@@ -0,0 +1,138 @@
<?php
namespace addons\shopro_huifu\library\onboard;
/**
* 联行号导入范围过滤（默认 main ≈ 5 万条：主要省市 + 四大行/农商行等常用银行）
*/
class HuifuOnboardBankBranchImportFilter
{
public static function resolvePreset($preset)
{
$preset = strtolower(trim((string) $preset));
if ($preset === '' || $preset === 'main') {
return self::mainPreset();
}
if ($preset === 'full') {
return ['enabled' => false, 'name' => 'full'];
}
throw new \InvalidArgumentException('未知 preset: ' . $preset . '，可选 main / full');
}
protected static function mainPreset()
{
return [
'enabled' => true,
'name' => 'main',
'description' => '主要省市 + 四大行/交邮/农商行等，约5万条',
'bank_keywords' => [
'工商银行',
'农业银行',
'中国银行',
'建设银行',
'交通银行',
'邮储银行',
'邮政储蓄',
'农村商业银行',
'农村合作银行',
'农村信用',
'农商行',
'信用社',
'村镇银行',
'农信',
],
'major_provinces' => [
'北京', '上海', '天津', '重庆',
'河北', '山西', '辽宁', '吉林', '黑龙江',
'江苏', '浙江', '安徽', '福建', '江西', '山东', '河南', '湖北', '湖南', '广东', '海南',
'四川', '贵州', '云南', '陕西', '甘肃',
'内蒙古', '广西', '宁夏', '新疆',
],
'major_cities' => [
'北京', '上海', '天津', '重庆',
'广州', '深圳', '东莞', '佛山', '珠海', '中山', '惠州', '汕头', '湛江',
'杭州', '宁波', '温州', '嘉兴', '金华', '绍兴', '台州',
'南京', '苏州', '无锡', '常州', '南通', '徐州',
'成都', '绵阳', '德阳',
'武汉', '宜昌', '襄阳',
'长沙', '株洲', '湘潭',
'郑州', '洛阳',
'西安', '咸阳',
'合肥', '芜湖',
'福州', '厦门', '泉州',
'济南', '青岛', '烟台', '潍坊',
'沈阳', '大连',
'哈尔滨', '长春',
'石家庄', '唐山', '保定',
'太原',
'南昌', '赣州',
'昆明', '曲靖',
'贵阳', '遵义', '安顺', '毕节', '铜仁',
'南宁', '桂林', '柳州',
'海口', '三亚',
'兰州',
'乌鲁木齐', '呼和浩特',
'银川',
'拉萨', '西宁',
'韶关', '江门', '肇庆', '清远', '茂名', '梅州',
],
'max_matched' => 55000,
];
}
public static function matches(array $row, array $preset)
{
if (empty($preset['enabled'])) {
return true;
}
if (!self::matchBank($row, $preset['bank_keywords'] ?? [])) {
return false;
}
return self::matchRegion($row, $preset);
}
protected static function matchBank(array $row, array $keywords)
{
$text = self::rowText($row, ['branch_name', 'LName', 'bank_name', 'BankName']);
foreach ($keywords as $keyword) {
if ($keyword !== '' && mb_strpos($text, $keyword, 0, 'UTF-8') !== false) {
return true;
}
}
return false;
}
protected static function matchRegion(array $row, array $preset)
{
$province = self::rowText($row, ['province', 'ProvinceName']);
$city = self::rowText($row, ['city', 'CityName']);
$branchName = self::rowText($row, ['branch_name', 'LName']);
foreach ($preset['major_provinces'] ?? [] as $name) {
if ($name !== '' && (mb_strpos($province, $name, 0, 'UTF-8') !== false || mb_strpos($branchName, $name, 0, 'UTF-8') !== false)) {
return true;
}
}
foreach ($preset['major_cities'] ?? [] as $name) {
if ($name !== '' && (mb_strpos($city, $name, 0, 'UTF-8') !== false || mb_strpos($branchName, $name, 0, 'UTF-8') !== false)) {
return true;
}
}
return false;
}
protected static function rowText(array $row, array $keys)
{
$parts = [];
foreach ($keys as $key) {
if (!empty($row[$key])) {
$parts[] = trim((string) $row[$key]);
}
}
return implode(' ', $parts);
}
}

293
addons/shopro_huifu/library/onboard/HuifuOnboardBankBranchImporter.php
查看文件

@@ -8,9 +8,6 @@ class HuifuOnboardBankBranchImporter
{
protected static $areaNameMap;
/** @var array<string, bool> */
protected static $existingCodes;
/** @var array<string, bool> */
protected static $seenCodes;
@@ -34,73 +31,195 @@ class HuifuOnboardBankBranchImporter
$dryRun = !empty($options['dry_run']);
$truncate = !empty($options['truncate']);
$updateExisting = !empty($options['update_existing']);
$preset = HuifuOnboardBankBranchImportFilter::resolvePreset($options['preset'] ?? 'main');
if ($truncate && !$dryRun) {
Db::name('shopro_huifu_bank_branch')->where('id', '>', 0)->delete();
}
self::$existingCodes = null;
self::$seenCodes = [];
if (!$dryRun) {
self::loadExistingCodes();
if ($format === 'json') {
$rows = self::parseJsonFile($file, $limit);
$stats = self::emptyImportStats();
$stats['parsed'] = count($rows);
return self::importRowIterator($rows, $stats, [
'batch' => $batch,
'dry_run' => $dryRun,
'update_existing' => $updateExisting,
'preset' => $preset,
]);
}
$rows = $format === 'json'
? self::parseJsonFile($file, $limit)
: self::parseCsvFile($file, $limit);
$inserted = 0;
$updated = 0;
$skippedInvalid = 0;
$skippedExisting = 0;
$duplicateInFile = 0;
$buffer = [];
$flushOptions = [
return self::importCsvStream($file, [
'limit' => $limit,
'batch' => $batch,
'dry_run' => $dryRun,
'update_existing' => $updateExisting,
];
'preset' => $preset,
]);
}
/**
* @param array{limit:int,batch:int,dry_run:bool,update_existing:bool} $options
*/
protected static function importCsvStream($file, array $options)
{
$handle = fopen($file, 'rb');
if (!$handle) {
throw new \RuntimeException('无法读取 CSV: ' . $file);
}
$limit = max(0, (int) ($options['limit'] ?? 0));
$header = null;
$stats = self::emptyImportStats();
$buffer = [];
$flushOptions = [
'dry_run' => !empty($options['dry_run']),
'update_existing' => !empty($options['update_existing']),
];
$batch = max(50, min(2000, (int) ($options['batch'] ?? 500)));
$preset = $options['preset'] ?? HuifuOnboardBankBranchImportFilter::resolvePreset('main');
$matched = 0;
$maxMatched = (int) ($preset['max_matched'] ?? 0);
while (($line = fgetcsv($handle)) !== false) {
if (!$line || self::isEmptyCsvLine($line)) {
continue;
}
if ($header === null) {
$header = self::normalizeHeader($line);
continue;
}
$stats['parsed']++;
$row = self::mapCsvLine($header, $line);
if (!$row) {
continue;
}
if (!HuifuOnboardBankBranchImportFilter::matches($row, $preset)) {
$stats['skipped_filtered']++;
if ($limit > 0 && $stats['parsed'] >= $limit) {
break;
}
continue;
}
$matched++;
if ($maxMatched > 0 && $matched > $maxMatched) {
break;
}
foreach ($rows as $row) {
$normalized = self::normalizeRow($row);
if (!$normalized) {
$skippedInvalid++;
continue;
$stats['skipped_invalid']++;
} elseif (isset(self::$seenCodes[$normalized['branch_code']])) {
$stats['duplicate_in_file']++;
} else {
self::$seenCodes[$normalized['branch_code']] = true;
$buffer[] = $normalized;
}
$code = $normalized['branch_code'];
if (isset(self::$seenCodes[$code])) {
$duplicateInFile++;
continue;
}
self::$seenCodes[$code] = true;
$buffer[] = $normalized;
if (count($buffer) >= $batch) {
$stats = self::flushBuffer($buffer, $flushOptions);
$inserted += $stats['inserted'];
$updated += $stats['updated'];
$skippedExisting += $stats['skipped_existing'];
self::mergeImportStats($stats, self::flushBuffer($buffer, $flushOptions));
$buffer = [];
}
if ($limit > 0 && $stats['parsed'] >= $limit) {
break;
}
}
fclose($handle);
if ($buffer) {
self::mergeImportStats($stats, self::flushBuffer($buffer, $flushOptions));
}
$stats['matched'] = $matched;
$stats['preset'] = $preset['name'] ?? 'main';
$stats['total'] = $stats['inserted'] + $stats['updated'];
return $stats;
}
/**
* @param array<int, array<string, mixed>> $rows
* @param array{batch:int,dry_run:bool,update_existing:bool,preset?:array} $options
*/
protected static function importRowIterator(array $rows, array $stats, array $options)
{
$buffer = [];
$batch = max(50, min(2000, (int) ($options['batch'] ?? 500)));
$flushOptions = [
'dry_run' => !empty($options['dry_run']),
'update_existing' => !empty($options['update_existing']),
];
$preset = $options['preset'] ?? HuifuOnboardBankBranchImportFilter::resolvePreset('main');
$matched = 0;
$maxMatched = (int) ($preset['max_matched'] ?? 0);
foreach ($rows as $row) {
if (!HuifuOnboardBankBranchImportFilter::matches($row, $preset)) {
$stats['skipped_filtered']++;
continue;
}
$matched++;
if ($maxMatched > 0 && $matched > $maxMatched) {
break;
}
$normalized = self::normalizeRow($row);
if (!$normalized) {
$stats['skipped_invalid']++;
continue;
}
if (isset(self::$seenCodes[$normalized['branch_code']])) {
$stats['duplicate_in_file']++;
continue;
}
self::$seenCodes[$normalized['branch_code']] = true;
$buffer[] = $normalized;
if (count($buffer) >= $batch) {
self::mergeImportStats($stats, self::flushBuffer($buffer, $flushOptions));
$buffer = [];
}
}
if ($buffer) {
$stats = self::flushBuffer($buffer, $flushOptions);
$inserted += $stats['inserted'];
$updated += $stats['updated'];
$skippedExisting += $stats['skipped_existing'];
self::mergeImportStats($stats, self::flushBuffer($buffer, $flushOptions));
}
$stats['matched'] = $matched;
$stats['preset'] = $preset['name'] ?? 'main';
$stats['total'] = $stats['inserted'] + $stats['updated'];
return $stats;
}
protected static function emptyImportStats()
{
return [
'inserted' => $inserted,
'updated' => $updated,
'skipped_invalid' => $skippedInvalid,
'skipped_existing' => $skippedExisting,
'duplicate_in_file' => $duplicateInFile,
'parsed' => count($rows),
'total' => $inserted + $updated,
'inserted' => 0,
'updated' => 0,
'skipped_invalid' => 0,
'skipped_existing' => 0,
'skipped_filtered' => 0,
'duplicate_in_file' => 0,
'matched' => 0,
'parsed' => 0,
'preset' => 'main',
'total' => 0,
];
}
protected static function mergeImportStats(array &$base, array $chunk)
{
foreach (['inserted', 'updated', 'skipped_existing'] as $key) {
$base[$key] += (int) ($chunk[$key] ?? 0);
}
}
/**
* 校验库内重复联行号 / 同名不同号
*
@@ -224,35 +343,6 @@ class HuifuOnboardBankBranchImporter
return $decoded;
}
protected static function parseCsvFile($file, $limit = 0)
{
$handle = fopen($file, 'rb');
if (!$handle) {
throw new \RuntimeException('无法读取 CSV: ' . $file);
}
$header = null;
$rows = [];
while (($line = fgetcsv($handle)) !== false) {
if (!$line || self::isEmptyCsvLine($line)) {
continue;
}
if ($header === null) {
$header = self::normalizeHeader($line);
continue;
}
$assoc = self::mapCsvLine($header, $line);
if ($assoc) {
$rows[] = $assoc;
}
if ($limit > 0 && count($rows) >= $limit) {
break;
}
}
fclose($handle);
return $rows;
}
protected static function normalizeHeader(array $header)
{
$map = [];
@@ -498,18 +588,6 @@ class HuifuOnboardBankBranchImporter
return strtolower($level . '|' . HuifuOnboardAreaHelper::toCode($provId) . '|' . $name);
}
protected static function loadExistingCodes()
{
if (self::$existingCodes !== null) {
return;
}
self::$existingCodes = [];
$codes = Db::name('shopro_huifu_bank_branch')->column('branch_code');
foreach ($codes ?: [] as $code) {
self::$existingCodes[(string) $code] = true;
}
}
/**
* @param array<int, array<string, mixed>> $rows
* @param array{dry_run?:bool,update_existing?:bool} $options
@@ -517,28 +595,23 @@ class HuifuOnboardBankBranchImporter
*/
protected static function flushBuffer(array $rows, array $options = [])
{
if (!$rows) {
return ['inserted' => 0, 'updated' => 0, 'skipped_existing' => 0];
}
$dryRun = !empty($options['dry_run']);
$updateExisting = !empty($options['update_existing']);
$inserted = 0;
$updated = 0;
$skippedExisting = 0;
if ($dryRun) {
foreach ($rows as $row) {
if (isset(self::$existingCodes[$row['branch_code']])) {
$skippedExisting++;
} else {
$inserted++;
}
}
return ['inserted' => $inserted, 'updated' => 0, 'skipped_existing' => $skippedExisting];
}
$codes = array_column($rows, 'branch_code');
$existingSet = self::lookupExistingCodes($codes);
$toInsert = [];
foreach ($rows as $row) {
$code = $row['branch_code'];
if (isset(self::$existingCodes[$code])) {
if ($updateExisting) {
$code = (string) $row['branch_code'];
if (isset($existingSet[$code])) {
if ($updateExisting && !$dryRun) {
Db::name('shopro_huifu_bank_branch')->where('branch_code', $code)->update([
'branch_name' => $row['branch_name'],
'bank_name' => $row['bank_name'],
@@ -552,8 +625,12 @@ class HuifuOnboardBankBranchImporter
}
continue;
}
if ($dryRun) {
$inserted++;
continue;
}
$toInsert[] = $row;
self::$existingCodes[$code] = true;
$existingSet[$code] = true;
}
if ($toInsert) {
@@ -564,6 +641,24 @@ class HuifuOnboardBankBranchImporter
return ['inserted' => $inserted, 'updated' => $updated, 'skipped_existing' => $skippedExisting];
}
/**
* @param array<int, string> $codes
* @return array<string, bool>
*/
protected static function lookupExistingCodes(array $codes)
{
$codes = array_values(array_unique(array_filter(array_map('strval', $codes))));
if (!$codes) {
return [];
}
$existing = Db::name('shopro_huifu_bank_branch')->where('branch_code', 'in', $codes)->column('branch_code');
$set = [];
foreach ($existing ?: [] as $code) {
$set[(string) $code] = true;
}
return $set;
}
protected static function downloadToFile($url, $dest)
{
if (function_exists('curl_init')) {