In [2]:
from huggingface_hub import Repository
In [10]:
%cd /workspace/llm-playground
/workspace/llm-playground
In [4]:
import os
HF_TOKEN = os.environ.get("HUGGINGFACE_TOKEN")
In [5]:
# https://huggingface.co/datasets/utensil/storage
repo = Repository(local_dir="storage", clone_from="utensil/storage", repo_type='dataset', skip_lfs_files=True, use_auth_token=HF_TOKEN)
/workspace/llm-playground/storage is already a clone of https://huggingface.co/datasets/utensil/storage. Make sure you pull the latest changes with `repo.git_pull()`.
In [6]:
repo.git_pull()
In [7]:
!date --rfc-3339=seconds > storage/time.txt
In [8]:
!cat storage/time.txt
2023-04-17 17:09:58+00:00
In [19]:
!pwd
/workspaces/llm-playground
In [9]:
!python helper/upload.py
Working directory changed to: /workspace/llm-playground/helper/.. /workspace/llm-playground/storage is already a clone of https://huggingface.co/datasets/utensil/storage. Make sure you pull the latest changes with `repo.git_pull()`. To https://huggingface.co/datasets/utensil/storage febf6ec..74662c3 main -> main Upload succeeded.
In [11]:
!ls /workspace/code/saved_models/llama-7b-hf_alpaca/
checkpoint-117 checkpoint-195 checkpoint-273 checkpoint-351 checkpoint-78 checkpoint-156 checkpoint-234 checkpoint-312 checkpoint-39
In [12]:
!mkdir -p storage/saved_models/llama-7b-hf_alpaca/
In [13]:
!cp -r /workspace/code/saved_models/llama-7b-hf_alpaca/checkpoint-351 storage/saved_models/llama-7b-hf_alpaca/
In [14]:
!ls storage/saved_models/llama-7b-hf_alpaca/
checkpoint-351
In [16]:
!ls -lhta storage/saved_models/llama-7b-hf_alpaca/checkpoint-351
total 49M drwxr-xr-x 2 root root 160 Apr 17 17:16 . -rw-r--r-- 1 root root 33M Apr 17 17:16 optimizer.pt -rw-r--r-- 1 root root 15K Apr 17 17:16 rng_state.pth -rw-r--r-- 1 root root 557 Apr 17 17:16 scaler.pt -rw-r--r-- 1 root root 627 Apr 17 17:16 scheduler.pt -rw-r--r-- 1 root root 4.2K Apr 17 17:16 trainer_state.json -rw-r--r-- 1 root root 17M Apr 17 17:16 pytorch_model.bin -rw-r--r-- 1 root root 3.5K Apr 17 17:16 training_args.bin drwxr-xr-x 3 root root 28 Apr 17 17:16 ..
In [17]:
!python helper/upload.py
Working directory changed to: /workspace/llm-playground/helper/.. /workspace/llm-playground/storage is already a clone of https://huggingface.co/datasets/utensil/storage. Make sure you pull the latest changes with `repo.git_pull()`. Upload file saved_models/llama-7b-hf_alpaca/checkpoint-351/optimizer.pt: 0%| | Upload file saved_models/llama-7b-hf_alpaca/checkpoint-351/training_args.bin: Upload file saved_models/llama-7b-hf_alpaca/checkpoint-351/pytorch_model.bin: Upload file saved_models/llama-7b-hf_alpaca/checkpoint-351/scheduler.pt: 0%| | Upload file saved_models/llama-7b-hf_alpaca/checkpoint-351/scaler.pt: 0%| | 1. Upload file saved_models/llama-7b-hf_alpaca/checkpoint-351/optimizer.pt: 48%|▍| Upload file saved_models/llama-7b-hf_alpaca/checkpoint-351/optimizer.pt: 95%|▉| Upload file saved_models/llama-7b-hf_alpaca/checkpoint-351/optimizer.pt: 45.8MB [00:04, 12.1MB/s]MB/s]To https://huggingface.co/datasets/utensil/storage 74662c3..68e82e4 main -> main Upload file saved_models/llama-7b-hf_alpaca/checkpoint-351/optimizer.pt: 100%|█| Upload file saved_models/llama-7b-hf_alpaca/checkpoint-351/training_args.bin: 10 Upload file saved_models/llama-7b-hf_alpaca/checkpoint-351/training_args.bin: 10 Upload file saved_models/llama-7b-hf_alpaca/checkpoint-351/pytorch_model.bin: 10 Upload file saved_models/llama-7b-hf_alpaca/checkpoint-351/scheduler.pt: 100%|█| Upload file saved_models/llama-7b-hf_alpaca/checkpoint-351/scheduler.pt: 100%|█| Upload file saved_models/llama-7b-hf_alpaca/checkpoint-351/scaler.pt: 100%|█| 55 Upload file saved_models/llama-7b-hf_alpaca/checkpoint-351/scaler.pt: 100%|█| 55 Upload file saved_models/llama-7b-hf_alpaca/checkpoint-351/rng_state.pth: 100%|█ Upload file saved_models/llama-7b-hf_alpaca/checkpoint-351/rng_state.pth: 100%|█ Upload succeeded.
In [18]:
!cp -r /workspace/code/saved_models/llama-7b-hf_alpaca/checkpoint-390 storage/saved_models/llama-7b-hf_alpaca/
In [19]:
!ls -lhta storage/saved_models/llama-7b-hf_alpaca/checkpoint-390
total 49M drwxr-xr-x 2 root root 160 Apr 17 17:34 . -rw-r--r-- 1 root root 33M Apr 17 17:34 optimizer.pt -rw-r--r-- 1 root root 15K Apr 17 17:34 rng_state.pth -rw-r--r-- 1 root root 557 Apr 17 17:34 scaler.pt -rw-r--r-- 1 root root 627 Apr 17 17:34 scheduler.pt -rw-r--r-- 1 root root 4.7K Apr 17 17:34 trainer_state.json -rw-r--r-- 1 root root 17M Apr 17 17:34 pytorch_model.bin -rw-r--r-- 1 root root 3.5K Apr 17 17:34 training_args.bin drwxr-xr-x 4 root root 50 Apr 17 17:34 ..
In [20]:
!cp /workspace/code/saved_models/llama-7b-hf_alpaca/adapter* storage/saved_models/llama-7b-hf_alpaca/
In [21]:
!ls -lhta storage/saved_models/llama-7b-hf_alpaca/
total 8.0K drwxr-xr-x 4 root root 102 Apr 17 17:35 . -rw-r--r-- 1 root root 350 Apr 17 17:35 adapter_config.json -rw-r--r-- 1 root root 443 Apr 17 17:35 adapter_model.bin drwxr-xr-x 2 root root 160 Apr 17 17:34 checkpoint-390 drwxr-xr-x 2 root root 160 Apr 17 17:16 checkpoint-351 drwxr-xr-x 3 root root 32 Apr 17 17:15 ..
In [22]:
!ls -lhta /workspace/code/saved_models/llama-7b-hf_alpaca/
total 12K drwxr-xr-x 12 root root 4.0K Apr 17 17:33 . -rw-r--r-- 1 root root 350 Apr 17 17:33 adapter_config.json -rw-r--r-- 1 root root 443 Apr 17 17:33 adapter_model.bin drwxr-xr-x 2 root root 160 Apr 17 17:33 checkpoint-390 drwxr-xr-x 2 root root 160 Apr 17 17:12 checkpoint-351 drwxr-xr-x 2 root root 160 Apr 17 16:51 checkpoint-312 drwxr-xr-x 2 root root 160 Apr 17 16:30 checkpoint-273 drwxr-xr-x 2 root root 160 Apr 17 16:10 checkpoint-234 drwxr-xr-x 2 root root 160 Apr 17 15:49 checkpoint-195 drwxr-xr-x 2 root root 160 Apr 17 15:28 checkpoint-156 drwxr-xr-x 2 root root 160 Apr 17 15:07 checkpoint-117 drwxr-xr-x 2 root root 160 Apr 17 14:46 checkpoint-78 drwxr-xr-x 2 root root 160 Apr 17 14:25 checkpoint-39 drwxr-xr-x 3 root root 32 Apr 17 14:05 ..
In [23]:
!python helper/upload.py
Working directory changed to: /workspace/llm-playground/helper/.. /workspace/llm-playground/storage is already a clone of https://huggingface.co/datasets/utensil/storage. Make sure you pull the latest changes with `repo.git_pull()`. Upload file saved_models/llama-7b-hf_alpaca/checkpoint-390/optimizer.pt: 0%| | Upload file saved_models/llama-7b-hf_alpaca/checkpoint-390/scheduler.pt: 0%| | Upload file saved_models/llama-7b-hf_alpaca/checkpoint-390/scaler.pt: 0%| | 1. Upload file saved_models/llama-7b-hf_alpaca/checkpoint-390/pytorch_model.bin: Upload file saved_models/llama-7b-hf_alpaca/checkpoint-390/rng_state.pth: 0%| Upload file saved_models/llama-7b-hf_alpaca/checkpoint-390/scaler.pt: 15.3MB [00:01, 16.0MB/s] Upload file saved_models/llama-7b-hf_alpaca/checkpoint-390/scheduler.pt: 15.3MB [00:01, 16.0MB/s] Upload file saved_models/llama-7b-hf_alpaca/checkpoint-390/rng_state.pth: 15.3MB [00:01, 16.0MB/s] Upload file saved_models/llama-7b-hf_alpaca/checkpoint-390/optimizer.pt: 48%|▍| Upload file saved_models/llama-7b-hf_alpaca/checkpoint-390/optimizer.pt: 45.8MB [00:05, 10.3MB/s]MB/s]To https://huggingface.co/datasets/utensil/storage 68e82e4..3b73be1 main -> main Upload file saved_models/llama-7b-hf_alpaca/checkpoint-390/optimizer.pt: 100%|█| Upload file saved_models/llama-7b-hf_alpaca/checkpoint-390/scheduler.pt: 100%|█| Upload file saved_models/llama-7b-hf_alpaca/checkpoint-390/scaler.pt: 100%|█| 55 Upload file saved_models/llama-7b-hf_alpaca/checkpoint-390/pytorch_model.bin: 10 Upload file saved_models/llama-7b-hf_alpaca/checkpoint-390/rng_state.pth: 100%|█ Upload succeeded.
In [ ]: