添加一页PPT总结：突出目的与意义
This commit is contained in:
freedakgmail
+126
-124
@@ -6,21 +6,20 @@
|
||||
# ./baidu-backup.sh # 自动使用 /年份/月日
|
||||
# ============================================================
|
||||
|
||||
# 不使用 set -e（与并行后台进程不兼容）
|
||||
set -e
|
||||
|
||||
# ---- 配置 ----
|
||||
APP_KEY="z3gemBZfg7KYj6U3eHNfIzTs7uYS9OMh"
|
||||
SECRET_KEY="ptCKj2DfxL0KtGR1pM08c9KO2t2UC7SR"
|
||||
TOKEN_FILE="$HOME/.baidu_pan_token.json"
|
||||
BLOCK_SIZE=$((32 * 1024 * 1024)) # 32MB 大分片减少HTTP开销
|
||||
PARALLEL=4 # 并行上传数
|
||||
BLOCK_SIZE=$((4 * 1024 * 1024)) # 4MB
|
||||
|
||||
# ---- 自动识别项目 ----
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
PROJECT_NAME="$(basename "$SCRIPT_DIR")"
|
||||
PARENT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
ZIP_FILE="/tmp/${PROJECT_NAME}.zip"
|
||||
MD5_DIR="/tmp/baidu_md5"
|
||||
MD5_FILE="/tmp/baidu_md5_list.txt"
|
||||
|
||||
# ---- 目标目录 ----
|
||||
if [ -n "$1" ]; then
|
||||
@@ -43,7 +42,7 @@ echo ""
|
||||
# 步骤1: 打包
|
||||
# ============================================================
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "[1/6] 📦 打包项目文件..."
|
||||
echo "[1/5] 📦 打包项目文件..."
|
||||
rm -f "$ZIP_FILE"
|
||||
cd "$PARENT_DIR"
|
||||
|
||||
@@ -76,7 +75,7 @@ echo " ✅ 打包完成: ${FILE_SIZE_MB}MB"
|
||||
# ============================================================
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "[2/6] 🔑 获取百度网盘授权..."
|
||||
echo "[2/5] 🔑 获取百度网盘授权..."
|
||||
|
||||
ACCESS_TOKEN=""
|
||||
|
||||
@@ -159,27 +158,12 @@ with open('$TOKEN_FILE', 'w') as f: json.dump(token, f)
|
||||
fi
|
||||
|
||||
# ============================================================
|
||||
# 步骤3: 创建远程目录
|
||||
# ============================================================
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "[3/6] 📁 创建远程目录 ${REMOTE_DIR} ..."
|
||||
DIR_RESP=$(curl -s "https://pan.baidu.com/rest/2.0/xpan/file?method=create&access_token=${ACCESS_TOKEN}" \
|
||||
-d "path=${REMOTE_DIR}&size=0&isdir=1")
|
||||
DIR_ERRNO=$(echo "$DIR_RESP" | python3 -c "import sys,json;print(json.load(sys.stdin).get('errno',99))" 2>/dev/null)
|
||||
if [ "$DIR_ERRNO" = "0" ] || [ "$DIR_ERRNO" = "-8" ]; then
|
||||
echo " ✅ 目录已就绪"
|
||||
else
|
||||
echo " ⚠️ 目录创建返回: errno=$DIR_ERRNO（继续尝试上传）"
|
||||
fi
|
||||
|
||||
# ============================================================
|
||||
# 步骤4: 预创建文件
|
||||
# 步骤3: 预创建文件
|
||||
# ============================================================
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
BLOCKS=$(( (FILE_SIZE + BLOCK_SIZE - 1) / BLOCK_SIZE ))
|
||||
echo "[4/6] 📋 预创建文件（${BLOCKS}个分片）..."
|
||||
echo "[3/5] 📋 预创建文件（${BLOCKS}个分片）..."
|
||||
BLOCK_LIST=$(python3 -c "import json; print(json.dumps(['0'*32]*$BLOCKS))")
|
||||
PRE_RESP=$(curl -s "https://pan.baidu.com/rest/2.0/xpan/file?method=precreate&access_token=${ACCESS_TOKEN}" \
|
||||
-d "path=${REMOTE_PATH}&size=${FILE_SIZE}&isdir=0&autoinit=1&block_list=${BLOCK_LIST}")
|
||||
@@ -192,134 +176,152 @@ fi
|
||||
echo " ✅ 预创建成功"
|
||||
|
||||
# ============================================================
|
||||
# 步骤5: 分片上传（32MB大分片 + 并行上传）
|
||||
# 步骤4: 分片上传（并发）
|
||||
# ============================================================
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "[5/6] 🚀 上传中（${PARALLEL}路并行, 每片$((BLOCK_SIZE/1024/1024))MB）..."
|
||||
PARALLEL="${BAIDU_PARALLEL:-8}"
|
||||
if [ "$PARALLEL" -gt "$BLOCKS" ]; then PARALLEL=$BLOCKS; fi
|
||||
echo "[4/5] 🚀 上传中（${PARALLEL} 路并发）..."
|
||||
echo ""
|
||||
|
||||
# 预切割文件
|
||||
SPLIT_DIR="/tmp/baidu_chunks"
|
||||
rm -rf "$SPLIT_DIR"
|
||||
mkdir -p "$SPLIT_DIR"
|
||||
split -b ${BLOCK_SIZE} -a 4 "$ZIP_FILE" "${SPLIT_DIR}/chunk_"
|
||||
CHUNK_FILES=($(ls "${SPLIT_DIR}/chunk_"* | sort))
|
||||
BLOCKS=${#CHUNK_FILES[@]}
|
||||
echo " 📊 已切割为 ${BLOCKS} 个分片"
|
||||
echo ""
|
||||
|
||||
# 创建md5结果目录
|
||||
MD5_DIR="/tmp/baidu_md5"
|
||||
rm -rf "$MD5_DIR"
|
||||
mkdir -p "$MD5_DIR"
|
||||
|
||||
UNSORTED_MD5="/tmp/baidu_md5_unsorted_${UPLOAD_ID}.txt"
|
||||
PROGRESS_FILE="/tmp/baidu_progress_${UPLOAD_ID}.txt"
|
||||
ERR_FILE="/tmp/baidu_err_${UPLOAD_ID}.txt"
|
||||
DONE_FLAG="/tmp/baidu_done_${UPLOAD_ID}"
|
||||
: > "$UNSORTED_MD5"
|
||||
: > "$PROGRESS_FILE"
|
||||
: > "$ERR_FILE"
|
||||
rm -f "$DONE_FLAG"
|
||||
: > "$MD5_FILE"
|
||||
START_TIME=$(date +%s)
|
||||
DONE_COUNT=0
|
||||
FAIL_FLAG="/tmp/baidu_upload_fail"
|
||||
rm -f "$FAIL_FLAG"
|
||||
|
||||
# 单片上传函数（带重试）
|
||||
upload_chunk() {
|
||||
local SEQ=$1
|
||||
local CHUNK_FILE=$2
|
||||
local RETRY=0
|
||||
local MAX_RETRY=3
|
||||
while [ $RETRY -lt $MAX_RETRY ]; do
|
||||
local RESP
|
||||
RESP=$(curl -s --connect-timeout 30 --max-time 300 \
|
||||
"https://d.pcs.baidu.com/rest/2.0/pcs/superfile2?method=upload&access_token=${ACCESS_TOKEN}&type=tmpfile&path=${REMOTE_PATH}&uploadid=${UPLOAD_ID}&partseq=${SEQ}" \
|
||||
-F "file=@${CHUNK_FILE}" 2>&1)
|
||||
local MD5
|
||||
MD5=$(echo "$RESP" | python3 -c "import sys,json;print(json.load(sys.stdin).get('md5',''))" 2>/dev/null)
|
||||
if [ -n "$MD5" ]; then
|
||||
echo "$MD5" > "${MD5_DIR}/$(printf '%04d' $SEQ).md5"
|
||||
return 0
|
||||
fi
|
||||
RETRY=$((RETRY + 1))
|
||||
if [ $RETRY -lt $MAX_RETRY ]; then
|
||||
sleep 2
|
||||
fi
|
||||
# 单分片上传 worker（并发安全：单行 echo < PIPE_BUF 是原子写）
|
||||
upload_part() {
|
||||
local i=$1
|
||||
local chunk="/tmp/baidu_chunk_${UPLOAD_ID}_${i}"
|
||||
dd if="$ZIP_FILE" bs="$BLOCK_SIZE" skip="$i" count=1 2>/dev/null > "$chunk"
|
||||
local resp md5 attempt
|
||||
md5=""
|
||||
for attempt in 1 2 3; do
|
||||
resp=$(curl -s --max-time 600 \
|
||||
"https://d.pcs.baidu.com/rest/2.0/pcs/superfile2?method=upload&access_token=${ACCESS_TOKEN}&type=tmpfile&path=${REMOTE_PATH}&uploadid=${UPLOAD_ID}&partseq=${i}" \
|
||||
-F "file=@${chunk}")
|
||||
md5=$(echo "$resp" | python3 -c "import sys,json;print(json.load(sys.stdin).get('md5',''))" 2>/dev/null)
|
||||
[ -n "$md5" ] && break
|
||||
sleep $((attempt * 2))
|
||||
done
|
||||
echo "chunk $SEQ failed: $RESP" > "${FAIL_FLAG}_${SEQ}"
|
||||
return 1
|
||||
}
|
||||
|
||||
# 并行上传
|
||||
for ((i=0; i<BLOCKS; i+=PARALLEL)); do
|
||||
PIDS=()
|
||||
for ((j=0; j<PARALLEL && i+j<BLOCKS; j++)); do
|
||||
SEQ=$((i + j))
|
||||
upload_chunk $SEQ "${CHUNK_FILES[$SEQ]}" &
|
||||
PIDS+=($!)
|
||||
done
|
||||
|
||||
# 等待本批完成
|
||||
BATCH_FAIL=0
|
||||
for PID in "${PIDS[@]}"; do
|
||||
wait $PID || BATCH_FAIL=1
|
||||
done
|
||||
if [ $BATCH_FAIL -ne 0 ]; then
|
||||
echo ""
|
||||
echo " ❌ 上传失败:"
|
||||
cat "${FAIL_FLAG}_"* 2>/dev/null
|
||||
rm -rf "$SPLIT_DIR" "$MD5_DIR" "$ZIP_FILE" "${FAIL_FLAG}_"*
|
||||
exit 1
|
||||
rm -f "$chunk"
|
||||
if [ -z "$md5" ]; then
|
||||
echo "part ${i} failed after 3 attempts: ${resp}" >> "$ERR_FILE"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# 计算进度
|
||||
DONE_COUNT=$((i + ${#PIDS[@]}))
|
||||
if [ $DONE_COUNT -gt $BLOCKS ]; then DONE_COUNT=$BLOCKS; fi
|
||||
PCT=$((DONE_COUNT * 100 / BLOCKS))
|
||||
UPLOADED_MB=$((DONE_COUNT * BLOCK_SIZE / 1024 / 1024))
|
||||
if [ $UPLOADED_MB -gt $FILE_SIZE_MB ]; then UPLOADED_MB=$FILE_SIZE_MB; fi
|
||||
NOW=$(date +%s)
|
||||
ELAPSED=$((NOW - START_TIME))
|
||||
if [ $ELAPSED -gt 0 ]; then
|
||||
SPEED_MB=$((UPLOADED_MB / ELAPSED))
|
||||
REMAINING_MB=$((FILE_SIZE_MB - UPLOADED_MB))
|
||||
if [ $SPEED_MB -gt 0 ]; then
|
||||
ETA=$((REMAINING_MB / SPEED_MB))
|
||||
ETA_STR="${ETA}s"
|
||||
if [ $ETA -ge 60 ]; then ETA_STR="$((ETA/60))m$((ETA%60))s"; fi
|
||||
echo "${i}|${md5}" >> "$UNSORTED_MD5"
|
||||
echo "x" >> "$PROGRESS_FILE"
|
||||
}
|
||||
export -f upload_part
|
||||
export ZIP_FILE BLOCK_SIZE ACCESS_TOKEN REMOTE_PATH UPLOAD_ID
|
||||
export UNSORTED_MD5 PROGRESS_FILE ERR_FILE
|
||||
|
||||
# 后台进度刷新进程（每秒刷新一次）
|
||||
(
|
||||
while :; do
|
||||
DONE=$(wc -l < "$PROGRESS_FILE" 2>/dev/null | tr -d ' ')
|
||||
DONE=${DONE:-0}
|
||||
if [ "$DONE" -gt "$BLOCKS" ]; then DONE=$BLOCKS; fi
|
||||
PCT=$((DONE * 100 / BLOCKS))
|
||||
UPLOADED_MB=$((DONE * BLOCK_SIZE / 1024 / 1024))
|
||||
if [ $UPLOADED_MB -gt $FILE_SIZE_MB ]; then UPLOADED_MB=$FILE_SIZE_MB; fi
|
||||
NOW=$(date +%s)
|
||||
ELAPSED=$((NOW - START_TIME))
|
||||
if [ $ELAPSED -gt 0 ] && [ $DONE -gt 0 ]; then
|
||||
SPEED_KB=$((DONE * BLOCK_SIZE / 1024 / ELAPSED))
|
||||
SPEED_INT=$((SPEED_KB / 1024))
|
||||
SPEED_DEC=$(( (SPEED_KB % 1024) * 10 / 1024 ))
|
||||
if [ "$SPEED_KB" -gt 0 ]; then
|
||||
REMAINING_KB=$(( (FILE_SIZE - DONE * BLOCK_SIZE) / 1024 ))
|
||||
if [ $REMAINING_KB -lt 0 ]; then REMAINING_KB=0; fi
|
||||
ETA=$(( REMAINING_KB / SPEED_KB ))
|
||||
ETA_MIN=$((ETA / 60))
|
||||
ETA_SEC=$((ETA % 60))
|
||||
ETA_STR="${ETA_MIN}m${ETA_SEC}s"
|
||||
else
|
||||
ETA_STR="计算中"
|
||||
fi
|
||||
else
|
||||
SPEED_INT=0
|
||||
SPEED_DEC=0
|
||||
ETA_STR="计算中"
|
||||
fi
|
||||
else
|
||||
SPEED_MB=0
|
||||
ETA_STR="计算中"
|
||||
fi
|
||||
|
||||
BAR_FILLED=$((PCT * 30 / 100))
|
||||
BAR_EMPTY=$((30 - BAR_FILLED))
|
||||
BAR=""
|
||||
for ((b=0; b<BAR_FILLED; b++)); do BAR="${BAR}█"; done
|
||||
for ((b=0; b<BAR_EMPTY; b++)); do BAR="${BAR}░"; done
|
||||
printf "\r %s %3d%% | %d/%dMB | %dMB/s | 剩余%s " "$BAR" "$PCT" "$UPLOADED_MB" "$FILE_SIZE_MB" "$SPEED_MB" "$ETA_STR"
|
||||
done
|
||||
BAR_FILLED=$((PCT * 30 / 100))
|
||||
BAR_EMPTY=$((30 - BAR_FILLED))
|
||||
BAR=""
|
||||
for ((b=0; b<BAR_FILLED; b++)); do BAR="${BAR}█"; done
|
||||
for ((b=0; b<BAR_EMPTY; b++)); do BAR="${BAR}░"; done
|
||||
printf "\r %s %3d%% | %d/%dMB | %d.%dMB/s | 剩余%s " "$BAR" "$PCT" "$UPLOADED_MB" "$FILE_SIZE_MB" "$SPEED_INT" "$SPEED_DEC" "$ETA_STR"
|
||||
if [ -f "$DONE_FLAG" ] || [ "$DONE" -ge "$BLOCKS" ]; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
) &
|
||||
PROGRESS_PID=$!
|
||||
|
||||
# 并发上传：xargs -P 同时跑 PARALLEL 个 worker（绕开百度 PCS 单连接限速）
|
||||
XARGS_RC=0
|
||||
seq 0 $((BLOCKS-1)) | xargs -P "$PARALLEL" -I % bash -c 'upload_part "$@"' _ % || XARGS_RC=$?
|
||||
|
||||
# 通知进度进程退出并等待，最后重画一行
|
||||
touch "$DONE_FLAG"
|
||||
wait "$PROGRESS_PID" 2>/dev/null || true
|
||||
echo ""
|
||||
|
||||
if [ "$XARGS_RC" -ne 0 ]; then
|
||||
echo ""
|
||||
echo " ❌ 分片上传失败："
|
||||
if [ -s "$ERR_FILE" ]; then
|
||||
head -n 5 "$ERR_FILE"
|
||||
fi
|
||||
rm -f "$ZIP_FILE" "$MD5_FILE" "$UNSORTED_MD5" "$PROGRESS_FILE" "$ERR_FILE" "$DONE_FLAG"
|
||||
rm -f /tmp/baidu_chunk_${UPLOAD_ID}_*
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 校验分片数
|
||||
DONE_COUNT=$(wc -l < "$UNSORTED_MD5" | tr -d ' ')
|
||||
if [ "$DONE_COUNT" != "$BLOCKS" ]; then
|
||||
echo ""
|
||||
echo " ❌ 分片数不匹配: ${DONE_COUNT}/${BLOCKS}"
|
||||
if [ -s "$ERR_FILE" ]; then
|
||||
head -n 5 "$ERR_FILE"
|
||||
fi
|
||||
rm -f "$ZIP_FILE" "$MD5_FILE" "$UNSORTED_MD5" "$PROGRESS_FILE" "$ERR_FILE" "$DONE_FLAG"
|
||||
rm -f /tmp/baidu_chunk_${UPLOAD_ID}_*
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 按 partseq 升序生成最终 MD5_FILE
|
||||
sort -t'|' -k1n "$UNSORTED_MD5" | cut -d'|' -f2 > "$MD5_FILE"
|
||||
rm -f "$UNSORTED_MD5" "$PROGRESS_FILE" "$ERR_FILE" "$DONE_FLAG"
|
||||
|
||||
echo ""
|
||||
echo " ✅ 所有分片上传完成"
|
||||
|
||||
# 删除临时分片
|
||||
rm -rf "$SPLIT_DIR"
|
||||
|
||||
# ============================================================
|
||||
# 步骤6: 合并文件
|
||||
# 步骤5: 合并文件
|
||||
# ============================================================
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "[6/6] 🔗 合并文件..."
|
||||
MD5_LIST=$(cat "${MD5_DIR}/"*.md5 | awk '{printf "\"%s\",", $0}' | sed 's/,$//')
|
||||
echo "[5/5] 🔗 合并文件..."
|
||||
MD5_LIST=$(awk '{printf "\"%s\",", $0}' "$MD5_FILE" | sed 's/,$//')
|
||||
CREATE_RESP=$(curl -s "https://pan.baidu.com/rest/2.0/xpan/file?method=create&access_token=${ACCESS_TOKEN}" \
|
||||
-d "path=${REMOTE_PATH}&size=${FILE_SIZE}&isdir=0&uploadid=${UPLOAD_ID}&block_list=[${MD5_LIST}]")
|
||||
CREATE_ERRNO=$(echo "$CREATE_RESP" | python3 -c "import sys,json;print(json.load(sys.stdin).get('errno',99))" 2>/dev/null)
|
||||
FINAL_PATH=$(echo "$CREATE_RESP" | python3 -c "import sys,json;print(json.load(sys.stdin).get('path',''))" 2>/dev/null)
|
||||
|
||||
# 清理
|
||||
rm -f "$ZIP_FILE" /tmp/chunk_part
|
||||
rm -rf "$MD5_DIR" "$FAIL_FLAG"
|
||||
rm -f "$ZIP_FILE" "$MD5_FILE"
|
||||
rm -f /tmp/baidu_chunk_${UPLOAD_ID}_* 2>/dev/null || true
|
||||
|
||||
# 计算耗时
|
||||
END_TIME=$(date +%s)
|
||||
|
||||
Reference in New Issue
Block a user