【modelscope/huggingface 通过colab将huggingface 模型/数据集/空间转移到 modelscope并下载】

1. 准备

注册一个modelscope账号（国内的）
拿到对应的访问令牌SDK/API令牌
注册一个google账号，登录colab

2. 开始干!

打开一个ipynb

安装依赖包

!pip install -qqq modelscope huggingface-hub -U

选择安装git lfs

!curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash!apt-get install git-lfs && git lfs install

转移模型

## 从huggingface下载模型/数据集/spaceimport json
from modelscope.hub.api import HubApi
from huggingface_hub import snapshot_download## huggingface
repo_id = "openai/clip-vit-large-patch14" # hf repo_id
repo_type = "model"  # model/dataset/space
local_dir = "clip-vit-large-patch14" # ## modelscope
framework = "Pytorch"
task = "image-generation"
username = "xxx" # modeoscope名字
ACCESS_TOKEN = "xxx-xxx-xxx-xxx-xxx" # modelscope访问令牌## 下载model
snapshot_download(repo_id,repo_type=repo_type,local_dir=local_dir,# allow_patterns=["*.bin", "*.json", "*.yaml", "*.txt", "*.ckpt"], # 选择哪些文件# ignore_patterns=[]  # 跳过哪些文件
)# 创建 configuration.json
with open(local_dir + "/configuration.json", "w", encoding="utf-8") as f:json.dump({"framework": framework, "task": task}, f, ensure_ascii=False, indent=4)## 开始上传文件
api = HubApi()
api.login(ACCESS_TOKEN)
model_id = username + "/" + "-".join(repo_id.split("/"))
api.push_model(model_id=model_id,  # 如果model_id对应的模型库不存在，将会自动创建model_dir=local_dir,  # 指定本地模型目录，目录中必须包含configuration.json文件
)
print(f'>>> model_id: {model_id}')
print(f'>>> 下载命令\nmodelscope download {model_id} --local_dir ./model_dir')