快速开始

conda create --name wallx python=3.10 -y
conda activate wallx

pip install -r requirements.txt
pip install "dmuon @ git+https://github.com/X-Square-Robot/dmuon.git"

git clone https://github.com/huggingface/lerobot.git
cd lerobot
git checkout v0.4.4
pip install --no-deps -e .
cd ../

export CUDA_HOME=/usr/local/cuda-12.8   # 改成你 nvcc 所在根目录，cuda 建议 12.8，和 torch 版本匹配
export PATH="$CUDA_HOME/bin:PATH"
which nvcc && nvcc --version
MAX_JOBS=8 pip install --no-build-isolation -e .

export FLASH_ATTN_CUDA_ARCHS=$(python -c 'import torch; print(f"{torch.cuda.get_device_capability()[0]}{torch.cuda.get_device_capability()[1]}")')
MAX_JOBS=4 pip install flash-attn==2.8.3 --no-build-isolation

export HF_ENDPOINT=https://hf-mirror.com
hf download <repo_id> --local-dir <your-path>

"""Download every file in a Hub repo without snapshot_download pagination.

hf-mirror's tree API returns absolute huggingface.co Link: next headers after
the first 1000 files. dataset_info().siblings is a complete path list and
hf_hub_download honors HF_ENDPOINT.

Usage:
    HF_ENDPOINT=https://hf-mirror.com HF_HUB_DISABLE_XET=1 HF_TOKEN=... \
      python workspace/download_hf_siblings.py \
        --repo-id ManipArena/maniparena-dataset \
        --repo-type dataset \
        --local-dir /path/to/dir
"""

from __future__ import annotations

import argparse
import os
import sys
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path

from huggingface_hub import dataset_info, hf_hub_download, model_info


def list_files(repo_id: str, repo_type: str) -> list[str]:
    info_fn = dataset_info if repo_type == "dataset" else model_info
    info = info_fn(repo_id)
    if not info.siblings:
        raise RuntimeError(f"{repo_id}: empty siblings list")
    return [s.rfilename for s in info.siblings]


def already_have(local_dir: Path, filename: str) -> bool:
    path = local_dir / filename
    return path.is_file() and path.stat().st_size > 0


def fetch(repo_id: str, repo_type: str, filename: str, local_dir: str, retries: int) -> str:
    last: Exception | None = None
    for attempt in range(retries):
        try:
            return hf_hub_download(
                repo_id,
                filename,
                repo_type=repo_type,
                local_dir=local_dir,
            )
        except Exception as exc:  # noqa: BLE001
            last = exc
            time.sleep(min(2**attempt, 16))
    raise RuntimeError(f"{filename}: {type(last).__name__} {last}") from last


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("--repo-id", required=True)
    parser.add_argument("--repo-type", default="dataset", choices=["dataset", "model"])
    parser.add_argument("--local-dir", required=True)
    parser.add_argument("--workers", type=int, default=8)
    parser.add_argument("--retries", type=int, default=4)
    args = parser.parse_args()

    local_dir = Path(args.local_dir)
    local_dir.mkdir(parents=True, exist_ok=True)

    files = list_files(args.repo_id, args.repo_type)
    pending = [f for f in files if not already_have(local_dir, f)]
    print(
        f"{args.repo_id}: {len(files)} files, {len(files) - len(pending)} already present, "
        f"{len(pending)} to fetch",
        flush=True,
    )
    if not pending:
        print(f"OK {args.repo_id} -> {local_dir}", flush=True)
        return

    done = 0
    errors: list[str] = []
    t0 = time.time()
    with ThreadPoolExecutor(max_workers=args.workers) as pool:
        futures = {
            pool.submit(fetch, args.repo_id, args.repo_type, f, str(local_dir), args.retries): f
            for f in pending
        }
        for future in as_completed(futures):
            try:
                future.result()
            except Exception as exc:  # noqa: BLE001
                errors.append(str(exc))
            done += 1
            if done % 100 == 0 or done == len(pending):
                dt = time.time() - t0
                print(
                    f"  {done}/{len(pending)}  errors={len(errors)}  {dt:.0f}s",
                    flush=True,
                )

    if errors:
        print(f"FAILED {len(errors)} files, first: {errors[0]}", flush=True)
        sys.exit(1)
    print(f"OK {args.repo_id} -> {local_dir}", flush=True)


if __name__ == "__main__":
    main()

huggingface-cli download X-Square-Robot/wall-oss-0.5 --local-dir /path/to/wall-oss-0.5

from huggingface_hub import snapshot_download

snapshot_download(
    "X-Square-Robot/wall-oss-0.5",
    local_dir="/path/to/wall-oss-0.5",
)

huggingface-cli download Qwen/Qwen2.5-VL-3B-Instruct --local-dir /path/to/Qwen2.5-VL-3B-Instruct