文件
xionggeshop/addons/shopro_huifu/library/onboard/HuifuOnboardBankBranchImporter.php
可乐加冰 29de6f63b7 ```
feat(shopro_huifu): 添加联行号导入范围筛选功能

- 新增 preset 参数支持 main(默认) 和 full 两种模式
- main 模式仅导入主要省市 + 四大行/农商行等约5万条数据
- full 模式导入全国全量约15万条数据
- 添加 HuifuOnboardBankBranchImportFilter 类处理筛选逻辑
- 优化 CSV 流式处理,提升大文件导入性能
- 更新 README 文档说明新的导入命令选项
```
2026-09-30 16:15:04 +08:00

702 行
23 KiB
PHP

<?php
namespace addons\shopro_huifu\library\onboard;
use think\Db;
class HuifuOnboardBankBranchImporter
{
protected static $areaNameMap;
/** @var array<string, bool> */
protected static $seenCodes;
/**
* @return array{inserted:int,updated:int,skipped_invalid:int,skipped_existing:int,duplicate_in_file:int,parsed:int,total:int}
*/
public static function importFile($file, array $options = [])
{
$file = (string) $file;
if (!is_file($file)) {
throw new \InvalidArgumentException('文件不存在: ' . $file);
}
$format = strtolower((string) ($options['format'] ?? 'auto'));
if ($format === 'auto') {
$format = self::detectFormat($file);
}
$limit = max(0, (int) ($options['limit'] ?? 0));
$batch = max(50, min(2000, (int) ($options['batch'] ?? 500)));
$dryRun = !empty($options['dry_run']);
$truncate = !empty($options['truncate']);
$updateExisting = !empty($options['update_existing']);
$preset = HuifuOnboardBankBranchImportFilter::resolvePreset($options['preset'] ?? 'main');
if ($truncate && !$dryRun) {
Db::name('shopro_huifu_bank_branch')->where('id', '>', 0)->delete();
}
self::$seenCodes = [];
if ($format === 'json') {
$rows = self::parseJsonFile($file, $limit);
$stats = self::emptyImportStats();
$stats['parsed'] = count($rows);
return self::importRowIterator($rows, $stats, [
'batch' => $batch,
'dry_run' => $dryRun,
'update_existing' => $updateExisting,
'preset' => $preset,
]);
}
return self::importCsvStream($file, [
'limit' => $limit,
'batch' => $batch,
'dry_run' => $dryRun,
'update_existing' => $updateExisting,
'preset' => $preset,
]);
}
/**
* @param array{limit:int,batch:int,dry_run:bool,update_existing:bool} $options
*/
protected static function importCsvStream($file, array $options)
{
$handle = fopen($file, 'rb');
if (!$handle) {
throw new \RuntimeException('无法读取 CSV: ' . $file);
}
$limit = max(0, (int) ($options['limit'] ?? 0));
$header = null;
$stats = self::emptyImportStats();
$buffer = [];
$flushOptions = [
'dry_run' => !empty($options['dry_run']),
'update_existing' => !empty($options['update_existing']),
];
$batch = max(50, min(2000, (int) ($options['batch'] ?? 500)));
$preset = $options['preset'] ?? HuifuOnboardBankBranchImportFilter::resolvePreset('main');
$matched = 0;
$maxMatched = (int) ($preset['max_matched'] ?? 0);
while (($line = fgetcsv($handle)) !== false) {
if (!$line || self::isEmptyCsvLine($line)) {
continue;
}
if ($header === null) {
$header = self::normalizeHeader($line);
continue;
}
$stats['parsed']++;
$row = self::mapCsvLine($header, $line);
if (!$row) {
continue;
}
if (!HuifuOnboardBankBranchImportFilter::matches($row, $preset)) {
$stats['skipped_filtered']++;
if ($limit > 0 && $stats['parsed'] >= $limit) {
break;
}
continue;
}
$matched++;
if ($maxMatched > 0 && $matched > $maxMatched) {
break;
}
$normalized = self::normalizeRow($row);
if (!$normalized) {
$stats['skipped_invalid']++;
} elseif (isset(self::$seenCodes[$normalized['branch_code']])) {
$stats['duplicate_in_file']++;
} else {
self::$seenCodes[$normalized['branch_code']] = true;
$buffer[] = $normalized;
}
if (count($buffer) >= $batch) {
self::mergeImportStats($stats, self::flushBuffer($buffer, $flushOptions));
$buffer = [];
}
if ($limit > 0 && $stats['parsed'] >= $limit) {
break;
}
}
fclose($handle);
if ($buffer) {
self::mergeImportStats($stats, self::flushBuffer($buffer, $flushOptions));
}
$stats['matched'] = $matched;
$stats['preset'] = $preset['name'] ?? 'main';
$stats['total'] = $stats['inserted'] + $stats['updated'];
return $stats;
}
/**
* @param array<int, array<string, mixed>> $rows
* @param array{batch:int,dry_run:bool,update_existing:bool,preset?:array} $options
*/
protected static function importRowIterator(array $rows, array $stats, array $options)
{
$buffer = [];
$batch = max(50, min(2000, (int) ($options['batch'] ?? 500)));
$flushOptions = [
'dry_run' => !empty($options['dry_run']),
'update_existing' => !empty($options['update_existing']),
];
$preset = $options['preset'] ?? HuifuOnboardBankBranchImportFilter::resolvePreset('main');
$matched = 0;
$maxMatched = (int) ($preset['max_matched'] ?? 0);
foreach ($rows as $row) {
if (!HuifuOnboardBankBranchImportFilter::matches($row, $preset)) {
$stats['skipped_filtered']++;
continue;
}
$matched++;
if ($maxMatched > 0 && $matched > $maxMatched) {
break;
}
$normalized = self::normalizeRow($row);
if (!$normalized) {
$stats['skipped_invalid']++;
continue;
}
if (isset(self::$seenCodes[$normalized['branch_code']])) {
$stats['duplicate_in_file']++;
continue;
}
self::$seenCodes[$normalized['branch_code']] = true;
$buffer[] = $normalized;
if (count($buffer) >= $batch) {
self::mergeImportStats($stats, self::flushBuffer($buffer, $flushOptions));
$buffer = [];
}
}
if ($buffer) {
self::mergeImportStats($stats, self::flushBuffer($buffer, $flushOptions));
}
$stats['matched'] = $matched;
$stats['preset'] = $preset['name'] ?? 'main';
$stats['total'] = $stats['inserted'] + $stats['updated'];
return $stats;
}
protected static function emptyImportStats()
{
return [
'inserted' => 0,
'updated' => 0,
'skipped_invalid' => 0,
'skipped_existing' => 0,
'skipped_filtered' => 0,
'duplicate_in_file' => 0,
'matched' => 0,
'parsed' => 0,
'preset' => 'main',
'total' => 0,
];
}
protected static function mergeImportStats(array &$base, array $chunk)
{
foreach (['inserted', 'updated', 'skipped_existing'] as $key) {
$base[$key] += (int) ($chunk[$key] ?? 0);
}
}
/**
* 校验库内重复联行号 / 同名不同号
*
* @return array{total:int,duplicate_codes:array,duplicate_names:array}
*/
public static function checkDatabaseDuplicates($nameSampleLimit = 10)
{
$nameSampleLimit = max(1, min(50, (int) $nameSampleLimit));
$total = (int) Db::name('shopro_huifu_bank_branch')->count();
$duplicateCodes = Db::name('shopro_huifu_bank_branch')
->field('branch_code, COUNT(*) AS cnt')
->group('branch_code')
->having('cnt > 1')
->select();
$duplicateNames = Db::query(
'SELECT branch_name, COUNT(*) AS cnt, COUNT(DISTINCT branch_code) AS code_cnt '
. 'FROM `' . config('database.prefix') . 'shopro_huifu_bank_branch` '
. 'GROUP BY branch_name HAVING cnt > 1 '
. 'ORDER BY cnt DESC LIMIT ' . $nameSampleLimit
);
return [
'total' => $total,
'duplicate_codes' => $duplicateCodes ?: [],
'duplicate_names' => $duplicateNames ?: [],
];
}
/**
* @return array{inserted:int,updated:int,skipped:int,total:int}
*/
public static function importSeedJson($jsonFile = null, array $options = [])
{
$jsonFile = $jsonFile ?: (ADDON_PATH . 'shopro_huifu/data/bank_branches.json');
return self::importFile($jsonFile, array_merge(['format' => 'json'], $options));
}
public static function downloadCnapsCsv($dest = null)
{
$dest = $dest ?: (ADDON_PATH . 'shopro_huifu/data/cnaps.csv');
$dir = dirname($dest);
if (!is_dir($dir)) {
mkdir($dir, 0755, true);
}
$urls = [
'https://hf-mirror.com/datasets/gweesin/CNAPS/resolve/main/cnaps.csv',
'https://huggingface.co/datasets/gweesin/CNAPS/resolve/main/cnaps.csv',
];
$lastError = '';
foreach ($urls as $url) {
try {
self::downloadToFile($url, $dest);
if (is_file($dest) && filesize($dest) > 1024) {
return $dest;
}
} catch (\Throwable $e) {
$lastError = $e->getMessage();
}
}
throw new \RuntimeException('下载全国联行号 CSV 失败,请手动下载后指定 --file。' . ($lastError ? ' ' . $lastError : ''));
}
public static function exportJson($dest, $limit = 0)
{
$dest = (string) $dest;
$limit = max(0, (int) $limit);
$query = Db::name('shopro_huifu_bank_branch')->where('status', 1)->order('id', 'asc');
if ($limit > 0) {
$query->limit($limit);
}
$list = $query->select();
$rows = [];
foreach ($list ?: [] as $item) {
$row = [
'branch_code' => (string) $item['branch_code'],
'branch_name' => (string) $item['branch_name'],
];
if (!empty($item['bank_name'])) {
$row['bank_name'] = (string) $item['bank_name'];
}
if (!empty($item['prov_id'])) {
$row['prov_id'] = (string) $item['prov_id'];
}
if (!empty($item['area_id'])) {
$row['area_id'] = (string) $item['area_id'];
}
$rows[] = $row;
}
$json = json_encode($rows, JSON_UNESCAPED_UNICODE | JSON_PRETTY_PRINT);
if ($json === false) {
throw new \RuntimeException('导出 JSON 失败');
}
file_put_contents($dest, $json . "\n");
return count($rows);
}
protected static function detectFormat($file)
{
$ext = strtolower(pathinfo($file, PATHINFO_EXTENSION));
if ($ext === 'json') {
return 'json';
}
return 'csv';
}
protected static function parseJsonFile($file, $limit = 0)
{
$decoded = json_decode(file_get_contents($file), true);
if (!is_array($decoded)) {
throw new \InvalidArgumentException('JSON 格式无效: ' . $file);
}
if ($limit > 0) {
return array_slice($decoded, 0, $limit);
}
return $decoded;
}
protected static function normalizeHeader(array $header)
{
$map = [];
foreach ($header as $idx => $name) {
$key = self::normalizeHeaderKey($name);
if ($key !== '') {
$map[$idx] = $key;
}
}
return $map;
}
protected static function normalizeHeaderKey($name)
{
$name = trim((string) $name);
$name = preg_replace('/^\xEF\xBB\xBF/', '', $name);
if ($name === '') {
return '';
}
$lower = strtolower($name);
$aliases = [
// gweesin/CNAPS: 联行号在 BankCode,BankId 是银行机构号(多支行共用,不可用)
'branch_code' => ['branch_code', 'bankcode', 'cnaps', 'cnapscode', '联行号', '行号', '开户行联行号', '大额支付行号'],
'branch_name' => ['branch_name', 'lname', 'banknamebranch', '支行名称', '开户行', '开户行支行名称', '支行全称', '网点名称'],
'bank_name' => ['bank_name', 'banknamemain', '银行', '银行名称', '银行简称', '总行名称'],
'province' => ['province', 'provincename', 'areaprovince', '省份', '省'],
'city' => ['city', 'cityname', 'areacity', '城市', '市', '地区'],
'prov_id' => ['prov_id', 'provincecode', '省码'],
'area_id' => ['area_id', '市码'],
];
foreach ($aliases as $canonical => $list) {
if (in_array($lower, $list, true) || in_array($name, $list, true)) {
return $canonical;
}
}
if ($lower === 'bankname') {
return 'bank_name';
}
if ($lower === 'citycode') {
return 'city_code';
}
if ($lower === 'bankid') {
return 'bank_inst_id';
}
return '';
}
protected static function mapCsvLine(array $header, array $line)
{
$row = [];
foreach ($header as $idx => $key) {
if (!array_key_exists($idx, $line)) {
continue;
}
$row[$key] = trim((string) $line[$idx]);
}
return $row;
}
protected static function isEmptyCsvLine(array $line)
{
foreach ($line as $cell) {
if (trim((string) $cell) !== '') {
return false;
}
}
return true;
}
protected static function normalizeRow(array $row)
{
$branchCode = self::normalizeBranchCode(self::pickBranchCode($row));
$branchName = trim((string) ($row['branch_name'] ?? ($row['LName'] ?? '')));
if ($branchCode === '' || $branchName === '') {
return null;
}
$bankName = trim((string) ($row['bank_name'] ?? ($row['BankName'] ?? '')));
if ($bankName === '') {
$bankName = self::guessBankName($branchName);
}
$provId = HuifuOnboardAreaHelper::toCode($row['prov_id'] ?? '');
$areaId = HuifuOnboardAreaHelper::toCode($row['area_id'] ?? '');
$province = trim((string) ($row['province'] ?? ($row['ProvinceName'] ?? '')));
$city = trim((string) ($row['city'] ?? ($row['CityName'] ?? '')));
if ($provId === '' && !empty($row['ProvinceCode'])) {
$provId = self::provinceCodeToAdmin((string) $row['ProvinceCode']);
}
if ($provId === '' && $province !== '') {
$provId = self::lookupAreaId($province, 1);
}
if ($areaId === '' && $city !== '') {
$areaId = self::lookupAreaId($city, 2, $provId);
}
if ($provId === '' && $areaId !== '') {
$provId = substr($areaId, 0, 2) . '0000';
}
return [
'branch_code' => $branchCode,
'branch_name' => $branchName,
'bank_name' => $bankName,
'prov_id' => $provId,
'area_id' => $areaId,
'status' => 1,
];
}
protected static function pickBranchCode(array $row)
{
foreach (['branch_code', 'BankCode', 'cnapsCode', '联行号'] as $key) {
if (!empty($row[$key])) {
return $row[$key];
}
}
return '';
}
protected static function normalizeBranchCode($code)
{
$code = preg_replace('/\D+/', '', (string) $code);
if ($code === '') {
return '';
}
if (strlen($code) < 8 || strlen($code) > 20) {
return '';
}
if (strlen($code) < 12) {
$code = str_pad($code, 12, '0', STR_PAD_LEFT);
}
return $code;
}
protected static function provinceCodeToAdmin($code)
{
$code = preg_replace('/\D+/', '', (string) $code);
if ($code === '') {
return '';
}
if (strlen($code) >= 6) {
return substr($code, 0, 2) . '0000';
}
return str_pad($code, 2, '0', STR_PAD_LEFT) . '0000';
}
protected static function guessBankName($branchName)
{
$rules = [
'工商银行' => '工商银行',
'农业银行' => '农业银行',
'中国银行' => '中国银行',
'建设银行' => '建设银行',
'交通银行' => '交通银行',
'招商银行' => '招商银行',
'浦发银行' => '浦发银行',
'民生银行' => '民生银行',
'光大银行' => '光大银行',
'兴业银行' => '兴业银行',
'平安银行' => '平安银行',
'中信银行' => '中信银行',
'华夏银行' => '华夏银行',
'广发银行' => '广发银行',
'邮储银行' => '邮储银行',
'邮政储蓄' => '邮储银行',
];
foreach ($rules as $needle => $short) {
if (mb_strpos($branchName, $needle, 0, 'UTF-8') !== false) {
return $short;
}
}
return '';
}
protected static function lookupAreaId($name, $level, $provId = '')
{
self::loadAreaNameMap();
$key = self::areaCacheKey($name, $level, $provId);
if (isset(self::$areaNameMap[$key])) {
return (string) self::$areaNameMap[$key];
}
$candidates = self::areaNameCandidates($name);
foreach ($candidates as $candidate) {
$tryKey = self::areaCacheKey($candidate, $level, $provId);
if (isset(self::$areaNameMap[$tryKey])) {
return (string) self::$areaNameMap[$tryKey];
}
}
return '';
}
protected static function loadAreaNameMap()
{
if (self::$areaNameMap !== null) {
return;
}
self::$areaNameMap = [];
try {
$rows = Db::name('shopro_area')->where('level', 'in', [1, 2])->field('id,name,pid,level')->select();
} catch (\Throwable $e) {
return;
}
foreach ($rows ?: [] as $row) {
$id = (string) $row['id'];
$level = (int) $row['level'];
$pid = (string) ($row['pid'] ?? '');
$provId = $level === 2 ? (substr($id, 0, 2) . '0000') : '';
foreach (self::areaNameCandidates($row['name']) as $candidate) {
$key = self::areaCacheKey($candidate, $level, $level === 2 ? $provId : '');
if (!isset(self::$areaNameMap[$key])) {
self::$areaNameMap[$key] = $id;
}
if ($level === 2 && $pid !== '' && $pid !== '0') {
$parentKey = self::areaCacheKey($candidate, $level, (string) $pid);
if (!isset(self::$areaNameMap[$parentKey])) {
self::$areaNameMap[$parentKey] = $id;
}
}
}
}
}
protected static function areaNameCandidates($name)
{
$name = trim((string) $name);
if ($name === '') {
return [];
}
$list = [$name];
$stripped = preg_replace('/(省|市|自治区|壮族|回族|维吾尔|特别行政区)$/u', '', $name);
if ($stripped && $stripped !== $name) {
$list[] = $stripped;
}
return array_values(array_unique($list));
}
protected static function areaCacheKey($name, $level, $provId = '')
{
return strtolower($level . '|' . HuifuOnboardAreaHelper::toCode($provId) . '|' . $name);
}
/**
* @param array<int, array<string, mixed>> $rows
* @param array{dry_run?:bool,update_existing?:bool} $options
* @return array{inserted:int,updated:int,skipped_existing:int}
*/
protected static function flushBuffer(array $rows, array $options = [])
{
if (!$rows) {
return ['inserted' => 0, 'updated' => 0, 'skipped_existing' => 0];
}
$dryRun = !empty($options['dry_run']);
$updateExisting = !empty($options['update_existing']);
$inserted = 0;
$updated = 0;
$skippedExisting = 0;
$codes = array_column($rows, 'branch_code');
$existingSet = self::lookupExistingCodes($codes);
$toInsert = [];
foreach ($rows as $row) {
$code = (string) $row['branch_code'];
if (isset($existingSet[$code])) {
if ($updateExisting && !$dryRun) {
Db::name('shopro_huifu_bank_branch')->where('branch_code', $code)->update([
'branch_name' => $row['branch_name'],
'bank_name' => $row['bank_name'],
'prov_id' => $row['prov_id'],
'area_id' => $row['area_id'],
'status' => 1,
]);
$updated++;
} else {
$skippedExisting++;
}
continue;
}
if ($dryRun) {
$inserted++;
continue;
}
$toInsert[] = $row;
$existingSet[$code] = true;
}
if ($toInsert) {
Db::name('shopro_huifu_bank_branch')->insertAll($toInsert);
$inserted = count($toInsert);
}
return ['inserted' => $inserted, 'updated' => $updated, 'skipped_existing' => $skippedExisting];
}
/**
* @param array<int, string> $codes
* @return array<string, bool>
*/
protected static function lookupExistingCodes(array $codes)
{
$codes = array_values(array_unique(array_filter(array_map('strval', $codes))));
if (!$codes) {
return [];
}
$existing = Db::name('shopro_huifu_bank_branch')->where('branch_code', 'in', $codes)->column('branch_code');
$set = [];
foreach ($existing ?: [] as $code) {
$set[(string) $code] = true;
}
return $set;
}
protected static function downloadToFile($url, $dest)
{
if (function_exists('curl_init')) {
$fp = fopen($dest, 'wb');
if (!$fp) {
throw new \RuntimeException('无法写入: ' . $dest);
}
$ch = curl_init($url);
curl_setopt_array($ch, [
CURLOPT_FILE => $fp,
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_TIMEOUT => 600,
CURLOPT_SSL_VERIFYPEER => true,
CURLOPT_USERAGENT => 'shopro_huifu-importer/1.0',
]);
$ok = curl_exec($ch);
$err = curl_error($ch);
$code = (int) curl_getinfo($ch, CURLINFO_HTTP_CODE);
curl_close($ch);
fclose($fp);
if (!$ok || $code >= 400) {
@unlink($dest);
throw new \RuntimeException('HTTP ' . $code . ($err ? ': ' . $err : ''));
}
return;
}
$ctx = stream_context_create([
'http' => [
'timeout' => 600,
'header' => "User-Agent: shopro_huifu-importer/1.0\r\n",
],
]);
$content = @file_get_contents($url, false, $ctx);
if ($content === false || $content === '') {
throw new \RuntimeException('下载失败: ' . $url);
}
file_put_contents($dest, $content);
}
}