安装部署
首先先安装ossutil工具,方便导出文件地址,安装地址:下载并安装命令行工具ossutil
下载成功后解压,然后可以看到最里面的文件夹里面有一个ossutil.exe文件,然后把这个文件的目录如D:\ossutil-2.4.1加入系统 Path 环境变量
去阿里云申请AccessKey密钥,然后打开cmd,输入下面指令进行配置
ossutil config跟着上面的操作,配置好就可以了
然后先进oss里面,去权限控制-读写权限-设置公共读
这样子准备工作就做好了
AccessKey密钥和oss公共读权限都是为了清除数据准备,清除后请设置成私有权限,并且删除AccessKey密钥,保障账号安全
拉取oss文件地址
进入D:\ossutil-2.4.1目录,我的是这个,你也可以换一个
文件夹地址栏输入cmd指令,输入以下指令,拉取oss的数据,即可拉取指定目录的文件地址
ossutil presign oss://Bucket名称/目录/ -r上面这个会把文件地址数据给输出到控制台
这个命令可以把文件输出到指定文件
ossutil.exe ls oss://Bucket名称/目录/ -r --short-format > file_list.txt --short-format这样子,就会把文件地址输出到"D:\ossutil-2.4.1\file_list.txt"文件
也可以拉取多个目录的文件地址
创建fetch_directory.bat文件,内容如下
@echo off
ossutil.exe ls oss://Bucket名称/目录/2026/0305/ -r --short-format >> file_list.txt
ossutil.exe ls oss://Bucket名称/目录/2026/0306/ -r --short-format >> file_list.txt
ossutil.exe ls oss://Bucket名称/目录/2026/0307/ -r --short-format >> file_list.txt然后双击运行就好了
脚本处理
先创建配置文件go.mod和go.sum
go.mod文件内容如下
module ossutil-uri-check
go 1.23.4
require (
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible
github.com/go-sql-driver/mysql v1.8.1
)
require (
filippo.io/edwards25519 v1.1.0 // indirect
golang.org/x/time v0.3.0 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
)
go.sum内容如下
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g=
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
创建check_missing_uri.go文件,内容如下
package main
import (
"bufio"
"context"
"database/sql"
"flag"
"fmt"
"os"
"path/filepath"
"regexp"
"strings"
"time"
mysql "github.com/go-sql-driver/mysql"
)
const queryChunkSize = 500
type databaseConfig struct {
Name string
Host string
Port int
DBName string
User string
Password string
Charset string
TablePrefix string
}
type uriState struct {
Active bool
Deleted bool
}
var tableNamePattern = regexp.MustCompile(`^[A-Za-z0-9_]+$`)
var targetTables = []string{
"表a",
"表b",
}
var databaseConfigs = map[string]databaseConfig{
"local": {
Name: "本地",
Host: "127.0.0.1",
Port: 3306,
DBName: "表名",
User: "用户名",
Password: "密码",
Charset: "utf8",
TablePrefix: "",
},
"local-zc": {
Name: "zc本地",
Host: "127.0.0.1",
Port: 3306,
DBName: "表名",
User: "用户名",
Password: "密码",
Charset: "utf8",
TablePrefix: "",
}
}
func main() {
if err := run(); err != nil {
fmt.Fprintln(os.Stderr, "执行失败:", err)
os.Exit(1)
}
}
func run() error {
databaseName := flag.String("db", "", "要检查的数据库:local、local-zc(必填)")
inputPath := flag.String("input", `D:\ossutil-2.4.1\file_list.txt`, "待检查的 URI 清单文件")
outputPath := flag.String("output", "", "检测结果输出文件;不指定时按数据库生成")
flag.Parse()
databaseKey, err := normalizeDatabaseName(*databaseName)
if err != nil {
flag.Usage()
return err
}
config, ok := databaseConfigs[databaseKey]
if !ok {
return fmt.Errorf("数据库配置不存在:%s", databaseKey)
}
resolvedOutputPath := *outputPath
if resolvedOutputPath == "" {
resolvedOutputPath = fmt.Sprintf("检测结果_%s.txt", databaseKey)
}
uris, err := readURIList(*inputPath)
if err != nil {
return err
}
fmt.Printf("=== %s数据库 ===\n", config.Name)
fmt.Printf("连接信息:%s@%s:%d/%s\n", config.User, config.Host, config.Port, config.DBName)
fmt.Printf("检测表:%s\n", strings.Join(targetTables, "、"))
states, err := findURIStates(uris, config)
if err != nil {
return fmt.Errorf("%s数据库:%w", config.Name, err)
}
notFound := make([]string, 0)
deleted := make([]string, 0)
activeCount := 0
for _, uri := range uris {
state, found := states[uri]
switch {
case !found:
notFound = append(notFound, uri)
case state.Active:
// 任一表中仍存在未删除记录,就视为有效数据,不进入已删除清单。
activeCount++
case state.Deleted:
deleted = append(deleted, uri)
default:
notFound = append(notFound, uri)
}
}
if err := writeReport(resolvedOutputPath, notFound, deleted); err != nil {
return err
}
outputAbsPath, err := filepath.Abs(resolvedOutputPath)
if err != nil {
outputAbsPath = resolvedOutputPath
}
fmt.Printf("输入 URI 数量:%d\n", len(uris))
fmt.Printf("没有找到:%d\n", len(notFound))
fmt.Printf("已经被删除:%d\n", len(deleted))
fmt.Printf("正常存在:%d\n", activeCount)
fmt.Printf("结果文件:%s\n", outputAbsPath)
return nil
}
func normalizeDatabaseName(value string) (string, error) {
switch strings.ToLower(strings.TrimSpace(value)) {
case "local", "本地":
return "local", nil
case "local-zc", "zc本地":
return "local-zc", nil
case "":
return "", fmt.Errorf("必须使用 -db 指定数据库(local、local-zc)")
default:
return "", fmt.Errorf("不支持的数据库 %q,请使用 local 或 local-zc", value)
}
}
// readURIList 读取清单,去除 UTF-8 BOM、空行和首尾空白,并按首次出现顺序去重。
func readURIList(path string) ([]string, error) {
file, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("无法读取输入文件 %s:%w", path, err)
}
defer file.Close()
scanner := bufio.NewScanner(file)
scanner.Buffer(make([]byte, 64*1024), 1024*1024)
uris := make([]string, 0)
seen := make(map[string]struct{})
firstLine := true
for scanner.Scan() {
line := scanner.Text()
if firstLine {
line = strings.TrimPrefix(line, "\uFEFF")
firstLine = false
}
uri := strings.TrimSpace(line)
if uri == "" {
continue
}
if _, ok := seen[uri]; ok {
continue
}
seen[uri] = struct{}{}
uris = append(uris, uri)
}
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("读取输入文件 %s 失败:%w", path, err)
}
return uris, nil
}
// findURIStates 查询两张表的 uri/is_del,汇总每个 URI 的存在和删除状态。
func findURIStates(uris []string, cfg databaseConfig) (map[string]uriState, error) {
for _, baseTable := range targetTables {
table := cfg.TablePrefix + baseTable
if !tableNamePattern.MatchString(table) {
return nil, fmt.Errorf("非法的表名:%s", table)
}
}
mysqlConfig := mysql.Config{
User: cfg.User,
Passwd: cfg.Password,
Net: "tcp",
Addr: fmt.Sprintf("%s:%d", cfg.Host, cfg.Port),
DBName: cfg.DBName,
Params: map[string]string{"charset": cfg.Charset},
Timeout: 5 * time.Second,
ReadTimeout: 30 * time.Second,
WriteTimeout: 30 * time.Second,
AllowNativePasswords: true,
}
db, err := sql.Open("mysql", mysqlConfig.FormatDSN())
if err != nil {
return nil, fmt.Errorf("初始化数据库连接失败:%w", err)
}
defer db.Close()
db.SetMaxOpenConns(1)
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
if err := db.PingContext(ctx); err != nil {
return nil, fmt.Errorf("连接数据库失败:%w", err)
}
states := make(map[string]uriState, len(uris))
for _, baseTable := range targetTables {
table := cfg.TablePrefix + baseTable
if err := collectTableStates(ctx, db, table, uris, states); err != nil {
return nil, err
}
}
return states, nil
}
func collectTableStates(
ctx context.Context,
db *sql.DB,
table string,
uris []string,
states map[string]uriState,
) error {
for start := 0; start < len(uris); start += queryChunkSize {
end := min(start+queryChunkSize, len(uris))
chunk := uris[start:end]
placeholders := strings.TrimSuffix(strings.Repeat("?,", len(chunk)), ",")
query := fmt.Sprintf(
"SELECT `uri`, `is_del` FROM `%s` WHERE BINARY `uri` IN (%s)",
table,
placeholders,
)
args := make([]any, len(chunk))
for i, uri := range chunk {
args[i] = uri
}
if err := collectTableChunk(ctx, db, table, query, args, states); err != nil {
return err
}
}
return nil
}
func collectTableChunk(
ctx context.Context,
db *sql.DB,
table string,
query string,
args []any,
states map[string]uriState,
) error {
rows, err := db.QueryContext(ctx, query, args...)
if err != nil {
return fmt.Errorf("查询 %s.uri 失败:%w", table, err)
}
defer rows.Close()
for rows.Next() {
var uri string
var isDel int64
if err := rows.Scan(&uri, &isDel); err != nil {
return fmt.Errorf("读取 %s 查询结果失败:%w", table, err)
}
state := states[uri]
if isDel == 1 {
state.Deleted = true
} else {
state.Active = true
}
states[uri] = state
}
if err := rows.Err(); err != nil {
return fmt.Errorf("遍历 %s 查询结果失败:%w", table, err)
}
return nil
}
func writeReport(path string, notFound, deleted []string) error {
var content strings.Builder
content.WriteString("未找到数据:\n")
for _, uri := range notFound {
content.WriteString(uri)
content.WriteByte('\n')
}
content.WriteString("\n已删除数据:\n")
for _, uri := range deleted {
content.WriteString(uri)
content.WriteByte('\n')
}
if err := os.WriteFile(path, []byte(content.String()), 0o644); err != nil {
return fmt.Errorf("无法写入输出文件 %s:%w", path, err)
}
return nil
}这个脚本配置了两个数据库,数据库配置记得改成自己的,然后是查询两个库里面的表a和表b的uri字段,有没有"D:\ossutil-2.4.1\file_list.txt"文件里面的数据。输出是,"D:\ossutil-2.4.1\file_list.txt"数据在表里面找不到或者被删除,被删除的依据是is_del的值为1
启动指令为go run check_missing_uri.go -db local,就是检查local库,执行完成后会输出一个检测结果_local.txt文件,里面就是检测的值
创建一个pending_delete_file.txt文件,把需要删除的数据写进这个文件
然后创建delete_oss_files目录,新目录里面创建main.go文件,内容如下
package main
import (
"bufio"
"errors"
"flag"
"fmt"
"os"
"strings"
"github.com/aliyun/aliyun-oss-go-sdk/oss"
)
const defaultEnvFile = `改成自己的后端项目的配置文件`
type ossConfig struct {
AccessKeyID string
AccessKeySecret string
Endpoint string
Bucket string
}
type objectEntry struct {
Line int
Key string
}
func main() {
if err := run(); err != nil {
fmt.Fprintln(os.Stderr, "执行失败:", err)
os.Exit(1)
}
}
func run() error {
inputPath := flag.String("file", "pending_delete_file.txt", "待删除 OSS Object Key 清单;删除成功的条目会从文件中移除")
envFile := flag.String("env-file", defaultEnvFile, "包含 ALIYUN_OSS 配置的 .env 文件")
execute := flag.Bool("execute", false, "实际执行删除;不指定时仅 dry-run")
failedOutput := flag.String("failed-output", "pending_delete_failed.txt", "删除失败的对象清单;留空则不写文件")
flag.Parse()
entries, err := readObjectEntries(*inputPath)
if err != nil {
return err
}
fmt.Printf("待处理对象数量:%d\n", len(entries))
if len(entries) == 0 {
fmt.Println("清单为空,未执行任何 OSS 操作。")
return nil
}
if err := validateObjectEntries(entries); err != nil {
return err
}
config, err := loadOSSConfig(*envFile)
if err != nil {
return err
}
client, err := oss.New(
config.Endpoint,
config.AccessKeyID,
config.AccessKeySecret,
oss.Timeout(10, 30),
)
if err != nil {
return fmt.Errorf("初始化 OSS 客户端失败:%w", err)
}
bucket, err := client.Bucket(config.Bucket)
if err != nil {
return fmt.Errorf("初始化 OSS Bucket 失败:%w", err)
}
fmt.Printf("OSS Endpoint:%s\n", config.Endpoint)
fmt.Printf("OSS Bucket:%s\n", config.Bucket)
if !*execute {
fmt.Println("当前为 DRY-RUN,只展示将删除的对象,不会调用 OSS 删除接口。")
for i, entry := range entries {
fmt.Printf("[DRY-RUN] [%d/%d] 行 %d:%s\n", i+1, len(entries), entry.Line, entry.Key)
}
fmt.Println("确认清单无误后,增加 -execute 参数执行实际删除。")
return nil
}
fmt.Println("开始逐条删除 OSS 对象...")
successCount := 0
failedEntries := make([]objectEntry, 0)
for i, entry := range entries {
fmt.Printf("[%d/%d] 行 %d:%s\n", i+1, len(entries), entry.Line, entry.Key)
// 与 safeDel 保持一致:不提前判断对象是否存在,删除失败时继续处理后续对象。
if err := bucket.DeleteObject(entry.Key); err != nil {
fmt.Fprintf(os.Stderr, " 删除失败:%v\n", err)
failedEntries = append(failedEntries, entry)
continue
}
successCount++
remainingEntries := make([]objectEntry, 0, len(failedEntries)+len(entries)-i-1)
remainingEntries = append(remainingEntries, failedEntries...)
remainingEntries = append(remainingEntries, entries[i+1:]...)
if err := writeObjectEntries(*inputPath, remainingEntries); err != nil {
return fmt.Errorf("OSS 对象已删除,但更新待删除清单失败:%w", err)
}
fmt.Println(" 删除成功,已从待删除清单移除")
}
if *failedOutput != "" {
if err := writeObjectEntries(*failedOutput, failedEntries); err != nil {
return err
}
}
fmt.Printf("删除完成:成功 %d 条,失败 %d 条。\n", successCount, len(failedEntries))
if len(failedEntries) > 0 {
if *failedOutput != "" {
fmt.Printf("失败清单:%s\n", *failedOutput)
}
return fmt.Errorf("%d 条 OSS 对象删除失败", len(failedEntries))
}
return nil
}
// readObjectEntries 按行读取对象 Key,去除 BOM、空行和首尾空白,保持原始顺序。
func readObjectEntries(path string) ([]objectEntry, error) {
file, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("无法读取待删除清单 %s:%w", path, err)
}
defer file.Close()
scanner := bufio.NewScanner(file)
scanner.Buffer(make([]byte, 64*1024), 1024*1024)
entries := make([]objectEntry, 0)
lineNumber := 0
firstLine := true
for scanner.Scan() {
lineNumber++
line := scanner.Text()
if firstLine {
line = strings.TrimPrefix(line, "\uFEFF")
firstLine = false
}
key := strings.TrimSpace(line)
if key == "" {
continue
}
entries = append(entries, objectEntry{Line: lineNumber, Key: key})
}
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("读取待删除清单 %s 失败:%w", path, err)
}
return entries, nil
}
// validateObjectEntries 在执行删除前统一校验,避免无效数据导致批量中途失败。
func validateObjectEntries(entries []objectEntry) error {
for _, entry := range entries {
lowerKey := strings.ToLower(entry.Key)
if strings.HasPrefix(lowerKey, "http://") ||
strings.HasPrefix(lowerKey, "https://") ||
strings.HasPrefix(lowerKey, "oss://") {
return fmt.Errorf("第 %d 行必须是 OSS Object Key,不能是完整 URL:%s", entry.Line, entry.Key)
}
}
return nil
}
func loadOSSConfig(envFile string) (ossConfig, error) {
fileValues := make(map[string]string)
if strings.TrimSpace(envFile) != "" {
content, err := os.ReadFile(envFile)
switch {
case err == nil:
fileValues, err = parseINISection(string(content), "ALIYUN_OSS")
if err != nil {
return ossConfig{}, fmt.Errorf("解析 OSS 配置失败:%w", err)
}
case errors.Is(err, os.ErrNotExist):
fmt.Fprintf(os.Stderr, "提示:配置文件不存在 %s,将尝试读取环境变量。\n", envFile)
default:
return ossConfig{}, fmt.Errorf("读取配置文件 %s 失败:%w", envFile, err)
}
}
config := ossConfig{
AccessKeyID: firstNonEmpty(
os.Getenv("OSS_ACCESS_KEY_ID"),
os.Getenv("ALIYUN_OSS_ACCESSKEYID"),
os.Getenv("ALIYUN_OSS_ACCESS_KEY_ID"),
fileValues["ACCESSKEYID"],
),
AccessKeySecret: firstNonEmpty(
os.Getenv("OSS_ACCESS_KEY_SECRET"),
os.Getenv("ALIYUN_OSS_ACCESSKEYSECRET"),
os.Getenv("ALIYUN_OSS_ACCESS_KEY_SECRET"),
fileValues["ACCESSKEYSECRET"],
),
Endpoint: firstNonEmpty(
os.Getenv("OSS_ENDPOINT"),
os.Getenv("ALIYUN_OSS_ENDPOINT"),
fileValues["ENDPOINT"],
),
Bucket: firstNonEmpty(
os.Getenv("OSS_BUCKET"),
os.Getenv("ALIYUN_OSS_BUCKET"),
fileValues["BUCKET"],
),
}
missing := make([]string, 0, 4)
if config.AccessKeyID == "" {
missing = append(missing, "AccessKeyId")
}
if config.AccessKeySecret == "" {
missing = append(missing, "AccessKeySecret")
}
if config.Endpoint == "" {
missing = append(missing, "Endpoint")
}
if config.Bucket == "" {
missing = append(missing, "Bucket")
}
if len(missing) > 0 {
return ossConfig{}, fmt.Errorf(
"缺少 OSS 配置:%s;请在 %s 的 [ALIYUN_OSS] 中配置,或设置对应环境变量",
strings.Join(missing, "、"),
envFile,
)
}
return config, nil
}
func parseINISection(content, targetSection string) (map[string]string, error) {
values := make(map[string]string)
currentSection := ""
lines := strings.Split(content, "\n")
for i, rawLine := range lines {
line := strings.TrimSpace(strings.TrimSuffix(rawLine, "\r"))
if line == "" || strings.HasPrefix(line, ";") || strings.HasPrefix(line, "#") {
continue
}
if strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]") {
currentSection = strings.TrimSpace(line[1 : len(line)-1])
continue
}
if !strings.EqualFold(currentSection, targetSection) {
continue
}
separator := strings.Index(line, "=")
if separator <= 0 {
return nil, fmt.Errorf("第 %d 行配置格式错误:%s", i+1, line)
}
key := strings.ToUpper(strings.TrimSpace(line[:separator]))
value := parseINIValue(strings.TrimSpace(line[separator+1:]))
if key == "" {
return nil, fmt.Errorf("第 %d 行配置键为空", i+1)
}
values[key] = value
}
return values, nil
}
func parseINIValue(value string) string {
if len(value) >= 2 {
quote := value[0]
if (quote == '"' || quote == '\'') && value[len(value)-1] == quote {
return value[1 : len(value)-1]
}
}
// 未加引号时,仅把空白后的 ; 或 # 视为行尾注释,避免误伤密钥中的符号。
for i := 1; i < len(value); i++ {
if value[i] != ';' && value[i] != '#' {
continue
}
if value[i-1] == ' ' || value[i-1] == '\t' {
return strings.TrimSpace(value[:i])
}
}
return value
}
func firstNonEmpty(values ...string) string {
for _, value := range values {
if value = strings.TrimSpace(value); value != "" {
return value
}
}
return ""
}
func writeObjectEntries(path string, entries []objectEntry) error {
var content strings.Builder
for _, entry := range entries {
content.WriteString(entry.Key)
content.WriteByte('\n')
}
if err := os.WriteFile(path, []byte(content.String()), 0o644); err != nil {
return fmt.Errorf("无法写入清单 %s:%w", path, err)
}
return nil
}
为了保障安全,因为是删除脚本,所以密钥信息我是放在项目的配置文件的,只给了项目目录地址,地址改成你自己的就行
启动命令为go run ./delete_oss_files -execute,执行后就可以删除数据了
补充
处理筛选数据时,偶尔需要处理空格或筛选指定数据。我是使用sublime来处理的
筛选数据:鼠标选中指定数据---alt+F3(选中全部指定内容)---ctrl+L(选中整行)---剪切粘贴到新文件
处理空行:ctrl+H---打开正则模式---查找内容填\n\s*\n,替换内容填\n---然后全部替换
