publish-public-prune (2589B)
1 #!/bin/sh 2 # Delete objects in the Bunny storage zone that publish-public uploaded but 3 # that no longer exist in ~/repos/www-public/ (orphans from remove-repo, 4 # flipping a repo private, or a www-public rebuild). 5 # 6 # Uses the publish-public manifest as the source of truth for "what's on 7 # Bunny" — avoids walking the Bunny storage API directory-by-directory. 8 # Trade-off: won't catch files that landed on Bunny outside publish-public 9 # (e.g. manual uploads). For the normal case that's fine. 10 set -eu 11 12 . "$HOME/repos/config.env" 2>/dev/null || true 13 : "${BUNNY_ZONE:?set BUNNY_ZONE in .env (laptop) and re-run push}" 14 15 KEY_FILE=$HOME/repos/bunny.key 16 if [ ! -r "$KEY_FILE" ]; then 17 echo "publish-public-prune: $KEY_FILE missing" >&2 18 exit 1 19 fi 20 KEY=$(cat "$KEY_FILE") 21 ENDPOINT=https://storage.bunnycdn.com/$BUNNY_ZONE 22 SRC=$HOME/repos/www-public 23 MANIFEST=$HOME/repos/.publish-state/manifest.tsv 24 25 if [ ! -s "$MANIFEST" ]; then 26 echo "publish-public-prune: no manifest — run publish-public first" >&2 27 exit 0 28 fi 29 30 UPLOADED=$(mktemp) 31 LOCAL=$(mktemp) 32 FAILED=$(mktemp) 33 trap 'rm -f "$UPLOADED" "$LOCAL" "$FAILED" "$FAILED.sorted"' EXIT 34 35 awk -F'\t' '{print $2}' "$MANIFEST" | LC_ALL=C sort -u > "$UPLOADED" 36 ( cd "$SRC" && find -L . -type f ) | sed 's|^\./||' | LC_ALL=C sort > "$LOCAL" 37 38 TODO=$(mktemp) 39 trap 'rm -f "$UPLOADED" "$LOCAL" "$FAILED" "$FAILED.sorted" "$TODO"' EXIT 40 LC_ALL=C comm -23 "$UPLOADED" "$LOCAL" > "$TODO" 41 42 n=$(wc -l < "$TODO" | tr -d ' ') 43 if [ "$n" -eq 0 ]; then 44 echo "publish-public-prune: nothing to delete" 45 exit 0 46 fi 47 echo "publish-public-prune: deleting $n orphans" 48 49 export ENDPOINT KEY FAILED 50 xargs -P 8 -I{} sh -c ' 51 f=$1 52 code=$(curl -sS -X DELETE "$ENDPOINT/$f" \ 53 -H "AccessKey: $KEY" -o /dev/null -w "%{http_code}") 54 case $code in 55 200|204|404) ;; 56 *) echo "DEL $code: $f" >&2; echo "$f" >> "$FAILED" ;; 57 esac 58 ' _ {} < "$TODO" 59 60 # Drop successfully-deleted paths from the manifest. Keep entries for paths 61 # whose DELETE failed so they get retried on the next run. 62 : > "$FAILED.sorted" 63 [ -s "$FAILED" ] && LC_ALL=C sort -u "$FAILED" > "$FAILED.sorted" 64 awk -F'\t' -v todo="$TODO" -v failed="$FAILED.sorted" ' 65 BEGIN { 66 while ((getline p < todo) > 0) orphan[p]=1 67 while ((getline p < failed) > 0) keep[p]=1 68 } 69 !orphan[$2] || keep[$2] 70 ' "$MANIFEST" > "$MANIFEST.new" 71 mv "$MANIFEST.new" "$MANIFEST" 72 73 if [ -s "$FAILED" ]; then 74 fail_n=$(wc -l < "$FAILED" | tr -d ' ') 75 echo "publish-public-prune: $((n - fail_n)) deleted, $fail_n failed" 76 exit 1 77 fi 78 echo "publish-public-prune: $n deleted"