feat(data): replace git LFS with S3 for test data downloads

.gitattributes

@@ -4,17 +4,13 @@
 # Ensure Python files always use LF for line endings.
 *.py text eol=lf
 # Treat designated file types as binary and do not alter their contents or line endings.
-*.png filter=lfs diff=lfs merge=lfs -text binary
-*.jpg filter=lfs diff=lfs merge=lfs -text binary
-*.jpeg filter=lfs diff=lfs merge=lfs -text binary
+*.png binary
+*.jpg binary
+*.jpeg binary
 *.ico binary
 *.pdf binary
-# Explicit LFS tracking for test files
-/data/.lfs/*.tar.gz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text binary
-*.mp4 filter=lfs diff=lfs merge=lfs -text binary
-*.mov filter=lfs diff=lfs merge=lfs -text binary
-*.gif filter=lfs diff=lfs merge=lfs -text binary
-*.foxe filter=lfs diff=lfs merge=lfs -text binary
-docs/capabilities/memory/assets/** filter=lfs diff=lfs merge=lfs -text
-docs/capabilities/memory/assets/.gitattributes -filter -diff -merge text
+*.onnx binary
+*.mp4 binary
+*.mov binary
+*.gif binary
+*.foxe binary

.gitignore

@@ -30,9 +30,8 @@ __pycache__
 
 .bash_history
 
-# Ignore all test data directories but allow compressed files
-/data/*
-!/data/.lfs/
+# Ignore all test data (archives are stored in S3, not git)
+/data/
 
 # node env (used by devcontainers cli)
 node_modules
@@ -43,9 +42,8 @@ package-lock.json
 dist/
 build/
 
-# Ignore data directory but keep .lfs subdirectory
-data/*
-!data/.lfs/
+# Ignore data directory (archives are stored in S3, not git)
+data/
 FastSAM-x.pt
 yolo11n.pt
 

bin/hooks/lfs_check

@@ -50,7 +50,7 @@ done
 if [ ${#new_data[@]} -gt 0 ]; then
    echo -e "${RED}✗${NC} New test data detected at /data:"
    echo -e "  ${GREEN}${new_data[@]}${NC}"
-   echo -e "\nEither delete or run ${GREEN}./bin/lfs_push${NC}"
-   echo -e "(lfs_push will compress the files into /data/.lfs/, upload to LFS, and add them to your commit)"
+   echo -e "\nEither delete or run ${GREEN}./bin/s3_push${NC}"
+   echo -e "(s3_push will compress the files into /data/.lfs/ and upload to S3)"
    exit 1
 fi

bin/s3_push

@@ -0,0 +1,76 @@
+!/bin/bash
+# Compresses directories/files in data/* into data/.lfs/<name>.tar.gz
+# Uploads to S3 (replaces bin/lfs_push)
+
+set -e
+
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m'
+
+S3_BUCKET="${DIMOS_DATA_S3_BUCKET:-dimos-github-lfs}"
+S3_PREFIX="${DIMOS_DATA_S3_PREFIX:-.lfs/}"
+AWS_PROFILE="${AWS_PROFILE:-dimensional-aws}"
+
+ROOT=$(git rev-parse --show-toplevel)
+cd "$ROOT"
+
+if [ ! -d "data/" ]; then
+    echo -e "${YELLOW}No data directory found, skipping.${NC}"
+    exit 0
+fi
+
+mkdir -p data/.lfs
+
+compressed_entries=()
+
+for entry_path in data/*; do
+    [ ! "$entry_path" ] && continue
+
+    entry_name=$(basename "$entry_path")
+
+    [ "$entry_name" = ".lfs" ] && continue
+
+    # Skip SQLite ephemeral sidecar files
+    case "$entry_name" in
+        *-wal|*-shm|*-journal) continue ;;
+    esac
+
+    compressed_file="data/.lfs/${entry_name}.tar.gz"
+
+    if [ -f "$compressed_file" ]; then
+        continue
+    fi
+
+    echo -e "  ${YELLOW}Compressing${NC} $entry_path -> $compressed_file"
+
+    entry_size=$(du -sh "$entry_path" | cut -f1)
+    echo -e "    Data size: ${YELLOW}$entry_size${NC}"
+
+    tar -czf "$compressed_file" \
+        --exclude='*.tmp' \
+        --exclude='*.temp' \
+        --exclude='.DS_Store' \
+        --exclude='Thumbs.db' \
+        --checkpoint=1000 \
+        --checkpoint-action=dot \
+        -C "data/" \
+        "$entry_name"
+
+    if [ $? -eq 0 ]; then
+        compressed_size=$(du -sh "$compressed_file" | cut -f1)
+        echo -e "  ${GREEN}✓${NC} Compressed $entry_name (${GREEN}$entry_size${NC} → ${GREEN}$compressed_size${NC})"
+        compressed_entries+=("$entry_name")
+    else
+        echo -e "  ${RED}✗${NC} Failed to compress $entry_name"
+        exit 1
+    fi
+done
+
+echo -e "\n${GREEN}Syncing data/.lfs/ to s3://${S3_BUCKET}/${S3_PREFIX}${NC}"
+aws s3 sync data/.lfs/ "s3://${S3_BUCKET}/${S3_PREFIX}" \
+    --profile "$AWS_PROFILE" \
+    --exclude ".gitkeep"
+
+echo -e "${GREEN}✓${NC} Upload complete"