In [8]:
%env DEBIAN_FRONTEND=noninteractive
env: DEBIAN_FRONTEND=noninteractive
In [2]:
!python -V
Python 3.8.10
In [3]:
!pip -V
pip 23.0.1 from /home/ubuntu/.local/lib/python3.8/site-packages/pip (python 3.8)
In [5]:
!whoami
ubuntu
In [10]:
!sudo apt install -yq python3.10
Reading package lists... Building dependency tree... Reading state information... The following additional packages will be installed: libpython3.10-minimal libpython3.10-stdlib python3.10-minimal Suggested packages: python3.10-venv binfmt-support The following NEW packages will be installed: libpython3.10-minimal libpython3.10-stdlib python3.10 python3.10-minimal 0 upgraded, 4 newly installed, 0 to remove and 28 not upgraded. Need to get 5225 kB of archives. After this operation, 20.2 MB of additional disk space will be used. Err:1 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu focal/main amd64 libpython3.10-minimal amd64 3.10.10-1+focal1 404 Not Found [IP: 185.125.190.52 443] Err:2 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu focal/main amd64 python3.10-minimal amd64 3.10.10-1+focal1 404 Not Found [IP: 185.125.190.52 443] Err:3 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu focal/main amd64 libpython3.10-stdlib amd64 3.10.10-1+focal1 404 Not Found [IP: 185.125.190.52 443] Err:4 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu focal/main amd64 python3.10 amd64 3.10.10-1+focal1 404 Not Found [IP: 185.125.190.52 443] E: Failed to fetch https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu/pool/main/p/python3.10/libpython3.10-minimal_3.10.10-1+focal1_amd64.deb 404 Not Found [IP: 185.125.190.52 443] E: Failed to fetch https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu/pool/main/p/python3.10/python3.10-minimal_3.10.10-1+focal1_amd64.deb 404 Not Found [IP: 185.125.190.52 443] E: Failed to fetch https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu/pool/main/p/python3.10/libpython3.10-stdlib_3.10.10-1+focal1_amd64.deb 404 Not Found [IP: 185.125.190.52 443] E: Failed to fetch https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu/pool/main/p/python3.10/python3.10_3.10.10-1+focal1_amd64.deb 404 Not Found [IP: 185.125.190.52 443] E: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing?
In [12]:
!sudo apt install -yq python3.9
!sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.9 1
!sudo update-alternatives --config python # pick 3.9 if given option
!python -V # should be 3.9
Reading package lists... Building dependency tree... Reading state information... The following additional packages will be installed: libpython3.9-minimal libpython3.9-stdlib python3.9-minimal Suggested packages: python3.9-venv binfmt-support The following NEW packages will be installed: libpython3.9-minimal libpython3.9-stdlib python3.9 python3.9-minimal 0 upgraded, 4 newly installed, 0 to remove and 28 not upgraded. Need to get 5023 kB of archives. After this operation, 19.7 MB of additional disk space will be used. Get:1 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu focal/main amd64 libpython3.9-minimal amd64 3.9.16-1+focal1 [804 kB] Get:2 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu focal/main amd64 python3.9-minimal amd64 3.9.16-1+focal1 [2062 kB] Get:3 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu focal/main amd64 libpython3.9-stdlib amd64 3.9.16-1+focal1 [1662 kB] Get:4 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu focal/main amd64 python3.9 amd64 3.9.16-1+focal1 [495 kB] Fetched 5023 kB in 6s (889 kB/s) 78Selecting previously unselected package libpython3.9-minimal:amd64. (Reading database ... 279084 files and directories currently installed.) Preparing to unpack .../libpython3.9-minimal_3.9.16-1+focal1_amd64.deb ... 7Progress: [ 0%] [..........................................................] 87Progress: [ 6%] [###.......................................................] 8Unpacking libpython3.9-minimal:amd64 (3.9.16-1+focal1) ... 7Progress: [ 12%] [######....................................................] 8Selecting previously unselected package python3.9-minimal. Preparing to unpack .../python3.9-minimal_3.9.16-1+focal1_amd64.deb ... 7Progress: [ 18%] [##########................................................] 8Unpacking python3.9-minimal (3.9.16-1+focal1) ... 7Progress: [ 24%] [#############.............................................] 8Selecting previously unselected package libpython3.9-stdlib:amd64. Preparing to unpack .../libpython3.9-stdlib_3.9.16-1+focal1_amd64.deb ... 7Progress: [ 29%] [#################.........................................] 8Unpacking libpython3.9-stdlib:amd64 (3.9.16-1+focal1) ... 7Progress: [ 35%] [####################......................................] 8Selecting previously unselected package python3.9. Preparing to unpack .../python3.9_3.9.16-1+focal1_amd64.deb ... 7Progress: [ 41%] [#######################...................................] 8Unpacking python3.9 (3.9.16-1+focal1) ... 7Progress: [ 47%] [###########################...............................] 8Setting up libpython3.9-minimal:amd64 (3.9.16-1+focal1) ... 7Progress: [ 53%] [##############################............................] 87Progress: [ 59%] [##################################........................] 8Setting up python3.9-minimal (3.9.16-1+focal1) ... 7Progress: [ 65%] [#####################################.....................] 87Progress: [ 71%] [########################################..................] 8Setting up libpython3.9-stdlib:amd64 (3.9.16-1+focal1) ... 7Progress: [ 76%] [############################################..............] 87Progress: [ 82%] [###############################################...........] 8Setting up python3.9 (3.9.16-1+focal1) ... 7Progress: [ 88%] [###################################################.......] 87Progress: [ 94%] [######################################################....] 8Processing triggers for man-db (2.9.1-1) ... Processing triggers for mime-support (3.64ubuntu1) ... 78update-alternatives: using /usr/bin/python3.9 to provide /usr/bin/python (python) in auto mode There is only one alternative in link group python (providing /usr/bin/python): /usr/bin/python3.9 Nothing to configure. Python 3.9.16
In [13]:
!wget https://bootstrap.pypa.io/get-pip.py
!python get-pip.py
--2023-06-03 15:42:40-- https://bootstrap.pypa.io/get-pip.py
Resolving bootstrap.pypa.io (bootstrap.pypa.io)... 151.101.192.175, 151.101.128.175, 151.101.64.175, ...
Connecting to bootstrap.pypa.io (bootstrap.pypa.io)|151.101.192.175|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 2578580 (2.5M) [text/x-python]
Saving to: ‘get-pip.py’
get-pip.py 100%[===================>] 2.46M --.-KB/s in 0.009s
2023-06-03 15:42:41 (284 MB/s) - ‘get-pip.py’ saved [2578580/2578580]
Defaulting to user installation because normal site-packages is not writeable
Collecting pip
Downloading pip-23.1.2-py3-none-any.whl (2.1 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 2.1/2.1 MB 148.7 MB/s eta 0:00:00
Installing collected packages: pip
Successfully installed pip-23.1.2
In [14]:
!pip3 install torch --index-url https://download.pytorch.org/whl/cu118
Defaulting to user installation because normal site-packages is not writeable Looking in indexes: https://download.pytorch.org/whl/cu118 Requirement already satisfied: torch in /usr/lib/python3/dist-packages (1.13.1)
In [15]:
!pip3 list|grep torch
torch 1.13.1 torchvision 0.14.1
In [ ]:
!pip3 install -U torch --index-url https://download.pytorch.org/whl/cu118
Defaulting to user installation because normal site-packages is not writeable Looking in indexes: https://download.pytorch.org/whl/cu118 Requirement already satisfied: torch in ./.local/lib/python3.9/site-packages (2.0.1+cu118) Requirement already satisfied: filelock in /usr/lib/python3/dist-packages (from torch) (3.0.12) Requirement already satisfied: typing-extensions in /usr/lib/python3/dist-packages (from torch) (3.10.0.2) Requirement already satisfied: sympy in ./.local/lib/python3.9/site-packages (from torch) (1.11.1) Requirement already satisfied: networkx in /usr/lib/python3/dist-packages (from torch) (2.4) Requirement already satisfied: jinja2 in /usr/lib/python3/dist-packages (from torch) (2.10.1) Requirement already satisfied: triton==2.0.0 in ./.local/lib/python3.9/site-packages (from torch) (2.0.0) Requirement already satisfied: cmake in ./.local/lib/python3.9/site-packages (from triton==2.0.0->torch) (3.25.0) Requirement already satisfied: lit in ./.local/lib/python3.9/site-packages (from triton==2.0.0->torch) (15.0.7) Requirement already satisfied: mpmath>=0.19 in ./.local/lib/python3.9/site-packages (from sympy->torch) (1.2.1)
In [18]:
!git clone https://github.com/OpenAccess-AI-Collective/axolotl
Cloning into 'axolotl'... remote: Enumerating objects: 2526, done. remote: Counting objects: 100% (1269/1269), done. remote: Compressing objects: 100% (385/385), done. remote: Total 2526 (delta 923), reused 1100 (delta 818), pack-reused 1257 Receiving objects: 100% (2526/2526), 1.30 MiB | 12.59 MiB/s, done. Resolving deltas: 100% (1562/1562), done.
In [19]:
%cd axolotl
/home/ubuntu/axolotl
In [20]:
!pip3 install -e .
Defaulting to user installation because normal site-packages is not writeable
Obtaining file:///home/ubuntu/axolotl
Preparing metadata (setup.py) ... done
Collecting transformers@ git+https://github.com/huggingface/transformers.git (from axolotl==0.1)
Cloning https://github.com/huggingface/transformers.git to /tmp/pip-install-mt8imf30/transformers_e7d69891845841ad93145f052bc3f2f5
Running command git clone --filter=blob:none --quiet https://github.com/huggingface/transformers.git /tmp/pip-install-mt8imf30/transformers_e7d69891845841ad93145f052bc3f2f5
Resolved https://github.com/huggingface/transformers.git to commit 539e2281cd97c35ef4122757f26c88f44115fa94
Installing build dependencies ... done
Getting requirements to build wheel ... done
Preparing metadata (pyproject.toml) ... done
Collecting PyYAML==6.0 (from axolotl==0.1)
Downloading PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (661 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 661.8/661.8 kB 85.3 MB/s eta 0:00:00
Collecting accelerate (from axolotl==0.1)
Downloading accelerate-0.19.0-py3-none-any.whl (219 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 219.1/219.1 kB 59.2 MB/s eta 0:00:00
Collecting addict (from axolotl==0.1)
Downloading addict-2.4.0-py3-none-any.whl (3.8 kB)
Collecting bert-score==0.3.13 (from axolotl==0.1)
Downloading bert_score-0.3.13-py3-none-any.whl (61 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 61.1/61.1 kB 20.0 MB/s eta 0:00:00
Collecting bitsandbytes>=0.39.0 (from axolotl==0.1)
Downloading bitsandbytes-0.39.0-py3-none-any.whl (92.2 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 92.2/92.2 MB 38.5 MB/s eta 0:00:0000:0100:01
Collecting datasets (from axolotl==0.1)
Downloading datasets-2.12.0-py3-none-any.whl (474 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 474.6/474.6 kB 102.6 MB/s eta 0:00:00
Collecting einops (from axolotl==0.1)
Downloading einops-0.6.1-py3-none-any.whl (42 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 42.2/42.2 kB 9.6 MB/s eta 0:00:00
Collecting evaluate==0.4.0 (from axolotl==0.1)
Downloading evaluate-0.4.0-py3-none-any.whl (81 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 81.4/81.4 kB 24.2 MB/s eta 0:00:00
Collecting fire (from axolotl==0.1)
Downloading fire-0.5.0.tar.gz (88 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 88.3/88.3 kB 41.2 MB/s eta 0:00:00
Preparing metadata (setup.py) ... done
Collecting rouge-score==0.1.2 (from axolotl==0.1)
Downloading rouge_score-0.1.2.tar.gz (17 kB)
Preparing metadata (setup.py) ... done
Collecting scikit-learn==1.2.2 (from axolotl==0.1)
Downloading scikit_learn-1.2.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (9.6 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 9.6/9.6 MB 153.4 MB/s eta 0:00:00a 0:00:01
Requirement already satisfied: scipy in /usr/lib/python3/dist-packages (from axolotl==0.1) (1.3.3)
Collecting sentencepiece (from axolotl==0.1)
Downloading sentencepiece-0.1.99-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.3/1.3 MB 145.8 MB/s eta 0:00:00
Collecting wandb (from axolotl==0.1)
Downloading wandb-0.15.3-py3-none-any.whl (2.0 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 2.0/2.0 MB 173.0 MB/s eta 0:00:00
Collecting xformers (from axolotl==0.1)
Downloading xformers-0.0.20-cp39-cp39-manylinux2014_x86_64.whl (109.1 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 109.1/109.1 MB 28.8 MB/s eta 0:00:0000:0100:01
Requirement already satisfied: torch>=1.0.0 in /home/ubuntu/.local/lib/python3.9/site-packages (from bert-score==0.3.13->axolotl==0.1) (2.0.1+cu118)
Collecting pandas>=1.0.1 (from bert-score==0.3.13->axolotl==0.1)
Downloading pandas-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.4 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 12.4/12.4 MB 175.9 MB/s eta 0:00:00a 0:00:01
Requirement already satisfied: numpy in /usr/lib/python3/dist-packages (from bert-score==0.3.13->axolotl==0.1) (1.17.4)
Requirement already satisfied: requests in /usr/lib/python3/dist-packages (from bert-score==0.3.13->axolotl==0.1) (2.22.0)
Collecting tqdm>=4.31.1 (from bert-score==0.3.13->axolotl==0.1)
Downloading tqdm-4.65.0-py3-none-any.whl (77 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 77.1/77.1 kB 21.0 MB/s eta 0:00:00
Requirement already satisfied: matplotlib in /usr/lib/python3/dist-packages (from bert-score==0.3.13->axolotl==0.1) (3.1.2)
Collecting packaging>=20.9 (from bert-score==0.3.13->axolotl==0.1)
Downloading packaging-23.1-py3-none-any.whl (48 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 48.9/48.9 kB 14.4 MB/s eta 0:00:00
Collecting dill (from evaluate==0.4.0->axolotl==0.1)
Downloading dill-0.3.6-py3-none-any.whl (110 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 110.5/110.5 kB 37.8 MB/s eta 0:00:00
Collecting xxhash (from evaluate==0.4.0->axolotl==0.1)
Downloading xxhash-3.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (212 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 212.2/212.2 kB 59.6 MB/s eta 0:00:00
Collecting multiprocess (from evaluate==0.4.0->axolotl==0.1)
Downloading multiprocess-0.70.14-py39-none-any.whl (132 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 132.9/132.9 kB 42.6 MB/s eta 0:00:00
Collecting fsspec[http]>=2021.05.0 (from evaluate==0.4.0->axolotl==0.1)
Downloading fsspec-2023.5.0-py3-none-any.whl (160 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 160.1/160.1 kB 48.5 MB/s eta 0:00:00
Collecting huggingface-hub>=0.7.0 (from evaluate==0.4.0->axolotl==0.1)
Downloading huggingface_hub-0.15.1-py3-none-any.whl (236 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 236.8/236.8 kB 64.0 MB/s eta 0:00:00
Collecting responses<0.19 (from evaluate==0.4.0->axolotl==0.1)
Downloading responses-0.18.0-py3-none-any.whl (38 kB)
Requirement already satisfied: absl-py in /usr/lib/python3/dist-packages (from rouge-score==0.1.2->axolotl==0.1) (0.15.0)
Collecting nltk (from rouge-score==0.1.2->axolotl==0.1)
Downloading nltk-3.8.1-py3-none-any.whl (1.5 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.5/1.5 MB 136.5 MB/s eta 0:00:00
Requirement already satisfied: six>=1.14.0 in /usr/lib/python3/dist-packages (from rouge-score==0.1.2->axolotl==0.1) (1.14.0)
Collecting joblib>=1.1.1 (from scikit-learn==1.2.2->axolotl==0.1)
Using cached joblib-1.2.0-py3-none-any.whl (297 kB)
Collecting threadpoolctl>=2.0.0 (from scikit-learn==1.2.2->axolotl==0.1)
Downloading threadpoolctl-3.1.0-py3-none-any.whl (14 kB)
Requirement already satisfied: psutil in /usr/lib/python3/dist-packages (from accelerate->axolotl==0.1) (5.5.1)
Collecting pyarrow>=8.0.0 (from datasets->axolotl==0.1)
Downloading pyarrow-12.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (39.0 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 39.0/39.0 MB 84.2 MB/s eta 0:00:0000:0100:01
Collecting aiohttp (from datasets->axolotl==0.1)
Downloading aiohttp-3.8.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.0 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.0/1.0 MB 120.7 MB/s eta 0:00:00
Requirement already satisfied: filelock in /usr/lib/python3/dist-packages (from transformers@ git+https://github.com/huggingface/transformers.git->axolotl==0.1) (3.0.12)
Collecting regex!=2019.12.17 (from transformers@ git+https://github.com/huggingface/transformers.git->axolotl==0.1)
Downloading regex-2023.5.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (769 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 769.0/769.0 kB 126.3 MB/s eta 0:00:00
Collecting tokenizers!=0.11.3,<0.14,>=0.11.1 (from transformers@ git+https://github.com/huggingface/transformers.git->axolotl==0.1)
Downloading tokenizers-0.13.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 7.8/7.8 MB 174.1 MB/s eta 0:00:0000:01
Collecting safetensors>=0.3.1 (from transformers@ git+https://github.com/huggingface/transformers.git->axolotl==0.1)
Downloading safetensors-0.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.3/1.3 MB 99.1 MB/s eta 0:00:00
Requirement already satisfied: termcolor in /usr/lib/python3/dist-packages (from fire->axolotl==0.1) (1.1.0)
Requirement already satisfied: Click!=8.0.0,>=7.0 in /usr/lib/python3/dist-packages (from wandb->axolotl==0.1) (7.0)
Collecting GitPython!=3.1.29,>=1.0.0 (from wandb->axolotl==0.1)
Downloading GitPython-3.1.31-py3-none-any.whl (184 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 184.3/184.3 kB 56.3 MB/s eta 0:00:00
Collecting sentry-sdk>=1.0.0 (from wandb->axolotl==0.1)
Downloading sentry_sdk-1.25.0-py2.py3-none-any.whl (206 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 206.5/206.5 kB 64.2 MB/s eta 0:00:00
Collecting docker-pycreds>=0.4.0 (from wandb->axolotl==0.1)
Downloading docker_pycreds-0.4.0-py2.py3-none-any.whl (9.0 kB)
Collecting pathtools (from wandb->axolotl==0.1)
Downloading pathtools-0.1.2.tar.gz (11 kB)
Preparing metadata (setup.py) ... done
Collecting setproctitle (from wandb->axolotl==0.1)
Downloading setproctitle-1.3.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (30 kB)
Requirement already satisfied: setuptools in /usr/lib/python3/dist-packages (from wandb->axolotl==0.1) (45.2.0)
Requirement already satisfied: appdirs>=1.4.3 in /usr/lib/python3/dist-packages (from wandb->axolotl==0.1) (1.4.3)
Requirement already satisfied: typing-extensions in /usr/lib/python3/dist-packages (from wandb->axolotl==0.1) (3.10.0.2)
Collecting protobuf!=4.21.0,<5,>=3.15.0 (from wandb->axolotl==0.1)
Downloading protobuf-4.23.2-cp37-abi3-manylinux2014_x86_64.whl (304 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 304.5/304.5 kB 84.1 MB/s eta 0:00:00
Collecting pyre-extensions==0.0.29 (from xformers->axolotl==0.1)
Downloading pyre_extensions-0.0.29-py3-none-any.whl (12 kB)
Collecting typing-inspect (from pyre-extensions==0.0.29->xformers->axolotl==0.1)
Downloading typing_inspect-0.9.0-py3-none-any.whl (8.8 kB)
Requirement already satisfied: sympy in /home/ubuntu/.local/lib/python3.9/site-packages (from torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (1.11.1)
Requirement already satisfied: networkx in /usr/lib/python3/dist-packages (from torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (2.4)
Requirement already satisfied: jinja2 in /usr/lib/python3/dist-packages (from torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (2.10.1)
Requirement already satisfied: triton==2.0.0 in /home/ubuntu/.local/lib/python3.9/site-packages (from torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (2.0.0)
Requirement already satisfied: cmake in /home/ubuntu/.local/lib/python3.9/site-packages (from triton==2.0.0->torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (3.25.0)
Requirement already satisfied: lit in /home/ubuntu/.local/lib/python3.9/site-packages (from triton==2.0.0->torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (15.0.7)
Requirement already satisfied: attrs>=17.3.0 in /usr/lib/python3/dist-packages (from aiohttp->datasets->axolotl==0.1) (19.3.0)
Collecting charset-normalizer<4.0,>=2.0 (from aiohttp->datasets->axolotl==0.1)
Downloading charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (199 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 199.2/199.2 kB 57.8 MB/s eta 0:00:00
Collecting multidict<7.0,>=4.5 (from aiohttp->datasets->axolotl==0.1)
Downloading multidict-6.0.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (114 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 114.2/114.2 kB 50.4 MB/s eta 0:00:00
Collecting async-timeout<5.0,>=4.0.0a3 (from aiohttp->datasets->axolotl==0.1)
Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)
Collecting yarl<2.0,>=1.0 (from aiohttp->datasets->axolotl==0.1)
Downloading yarl-1.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (269 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 269.4/269.4 kB 73.9 MB/s eta 0:00:00
Collecting frozenlist>=1.1.1 (from aiohttp->datasets->axolotl==0.1)
Downloading frozenlist-1.3.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (158 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 158.8/158.8 kB 48.8 MB/s eta 0:00:00
Collecting aiosignal>=1.1.2 (from aiohttp->datasets->axolotl==0.1)
Downloading aiosignal-1.3.1-py3-none-any.whl (7.6 kB)
Collecting gitdb<5,>=4.0.1 (from GitPython!=3.1.29,>=1.0.0->wandb->axolotl==0.1)
Downloading gitdb-4.0.10-py3-none-any.whl (62 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 62.7/62.7 kB 16.4 MB/s eta 0:00:00
Collecting python-dateutil>=2.8.2 (from pandas>=1.0.1->bert-score==0.3.13->axolotl==0.1)
Using cached python_dateutil-2.8.2-py2.py3-none-any.whl (247 kB)
Collecting pytz>=2020.1 (from pandas>=1.0.1->bert-score==0.3.13->axolotl==0.1)
Downloading pytz-2023.3-py2.py3-none-any.whl (502 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 502.3/502.3 kB 104.5 MB/s eta 0:00:00
Collecting tzdata>=2022.1 (from pandas>=1.0.1->bert-score==0.3.13->axolotl==0.1)
Downloading tzdata-2023.3-py2.py3-none-any.whl (341 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 341.8/341.8 kB 82.8 MB/s eta 0:00:00
Collecting numpy (from bert-score==0.3.13->axolotl==0.1)
Downloading numpy-1.24.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (17.3 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 17.3/17.3 MB 119.4 MB/s eta 0:00:0000:0100:01
Collecting urllib3>=1.25.10 (from responses<0.19->evaluate==0.4.0->axolotl==0.1)
Downloading urllib3-2.0.2-py3-none-any.whl (123 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 123.2/123.2 kB 55.2 MB/s eta 0:00:00
Requirement already satisfied: certifi in /usr/lib/python3/dist-packages (from sentry-sdk>=1.0.0->wandb->axolotl==0.1) (2019.11.28)
Collecting smmap<6,>=3.0.1 (from gitdb<5,>=4.0.1->GitPython!=3.1.29,>=1.0.0->wandb->axolotl==0.1)
Downloading smmap-5.0.0-py3-none-any.whl (24 kB)
Requirement already satisfied: idna>=2.0 in /usr/lib/python3/dist-packages (from yarl<2.0,>=1.0->aiohttp->datasets->axolotl==0.1) (2.8)
Requirement already satisfied: mpmath>=0.19 in /home/ubuntu/.local/lib/python3.9/site-packages (from sympy->torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (1.2.1)
Collecting mypy-extensions>=0.3.0 (from typing-inspect->pyre-extensions==0.0.29->xformers->axolotl==0.1)
Downloading mypy_extensions-1.0.0-py3-none-any.whl (4.7 kB)
Building wheels for collected packages: rouge-score, transformers, fire, pathtools
Building wheel for rouge-score (setup.py) ... done
Created wheel for rouge-score: filename=rouge_score-0.1.2-py3-none-any.whl size=24954 sha256=8648fe266bbf97c65a9f3e675f9f094ad480623064e182ca46be38642a389e61
Stored in directory: /home/ubuntu/.cache/pip/wheels/9b/3d/39/09558097d3119ca0a4d462df68f22c6f3c1b345ac63a09b86e
Building wheel for transformers (pyproject.toml) ... done
Created wheel for transformers: filename=transformers-4.30.0.dev0-py3-none-any.whl size=7160079 sha256=d907e1ad91a8145e286873787b759255627d08029167f9092958066a94b6567c
Stored in directory: /tmp/pip-ephem-wheel-cache-p71kyqo1/wheels/f7/92/8c/752ff3bfcd3439805d8bbf641614da38ef3226e127ebea86ee
Building wheel for fire (setup.py) ... done
Created wheel for fire: filename=fire-0.5.0-py2.py3-none-any.whl size=116936 sha256=a8b9f22c736b704dd34da9a6ac343b725ac2b90031fd9930df10816437ac70cc
Stored in directory: /home/ubuntu/.cache/pip/wheels/f7/f1/89/b9ea2bf8f80ec027a88fef1d354b3816b4d3d29530988972f6
Building wheel for pathtools (setup.py) ... done
Created wheel for pathtools: filename=pathtools-0.1.2-py3-none-any.whl size=8784 sha256=041d16a945a1361c1d8e2d5323368993d62f317ce7225d098485ab16338c9e2f
Stored in directory: /home/ubuntu/.cache/pip/wheels/b7/0a/67/ada2a22079218c75a88361c0782855cc72aebc4d18d0289d05
Successfully built rouge-score transformers fire pathtools
Installing collected packages: tokenizers, sentencepiece, safetensors, pytz, pathtools, bitsandbytes, addict, xxhash, urllib3, tzdata, tqdm, threadpoolctl, smmap, setproctitle, regex, PyYAML, python-dateutil, protobuf, packaging, numpy, mypy-extensions, multidict, joblib, fsspec, frozenlist, fire, einops, docker-pycreds, dill, charset-normalizer, async-timeout, yarl, typing-inspect, sentry-sdk, scikit-learn, responses, pyarrow, pandas, nltk, multiprocess, huggingface-hub, gitdb, aiosignal, transformers, rouge-score, pyre-extensions, GitPython, aiohttp, wandb, datasets, evaluate, xformers, bert-score, accelerate, axolotl
Running setup.py develop for axolotl
Successfully installed GitPython-3.1.31 PyYAML-6.0 accelerate-0.19.0 addict-2.4.0 aiohttp-3.8.4 aiosignal-1.3.1 async-timeout-4.0.2 axolotl bert-score-0.3.13 bitsandbytes-0.39.0 charset-normalizer-3.1.0 datasets-2.12.0 dill-0.3.6 docker-pycreds-0.4.0 einops-0.6.1 evaluate-0.4.0 fire-0.5.0 frozenlist-1.3.3 fsspec-2023.5.0 gitdb-4.0.10 huggingface-hub-0.15.1 joblib-1.2.0 multidict-6.0.4 multiprocess-0.70.14 mypy-extensions-1.0.0 nltk-3.8.1 numpy-1.24.3 packaging-23.1 pandas-2.0.2 pathtools-0.1.2 protobuf-4.23.2 pyarrow-12.0.0 pyre-extensions-0.0.29 python-dateutil-2.8.2 pytz-2023.3 regex-2023.5.5 responses-0.18.0 rouge-score-0.1.2 safetensors-0.3.1 scikit-learn-1.2.2 sentencepiece-0.1.99 sentry-sdk-1.25.0 setproctitle-1.3.2 smmap-5.0.0 threadpoolctl-3.1.0 tokenizers-0.13.3 tqdm-4.65.0 transformers-4.30.0.dev0 typing-inspect-0.9.0 tzdata-2023.3 urllib3-2.0.2 wandb-0.15.3 xformers-0.0.20 xxhash-3.2.0 yarl-1.9.2
In [21]:
!accelerate config --config_file configs/accelerate/default_config.yaml default
/usr/lib/python3/dist-packages/requests/__init__.py:89: RequestsDependencyWarning: urllib3 (2.0.2) or chardet (3.0.4) doesn't match a supported version!
warnings.warn("urllib3 ({}) or chardet ({}) doesn't match a supported "
Traceback (most recent call last):
File "/home/ubuntu/.local/bin/accelerate", line 5, in <module>
from accelerate.commands.accelerate_cli import main
File "/home/ubuntu/.local/lib/python3.9/site-packages/accelerate/__init__.py", line 3, in <module>
from .accelerator import Accelerator
File "/home/ubuntu/.local/lib/python3.9/site-packages/accelerate/accelerator.py", line 39, in <module>
from .tracking import LOGGER_TYPE_TO_CLASS, GeneralTracker, filter_trackers
File "/home/ubuntu/.local/lib/python3.9/site-packages/accelerate/tracking.py", line 42, in <module>
from torch.utils import tensorboard
File "/home/ubuntu/.local/lib/python3.9/site-packages/torch/utils/tensorboard/__init__.py", line 12, in <module>
from .writer import FileWriter, SummaryWriter # noqa: F401
File "/home/ubuntu/.local/lib/python3.9/site-packages/torch/utils/tensorboard/writer.py", line 9, in <module>
from tensorboard.compat.proto.event_pb2 import SessionLog
File "/usr/lib/python3/dist-packages/tensorboard/compat/proto/event_pb2.py", line 15, in <module>
from tensorboard.compat.proto import summary_pb2 as tensorboard_dot_compat_dot_proto_dot_summary__pb2
File "/usr/lib/python3/dist-packages/tensorboard/compat/proto/summary_pb2.py", line 15, in <module>
from tensorboard.compat.proto import histogram_pb2 as tensorboard_dot_compat_dot_proto_dot_histogram__pb2
File "/usr/lib/python3/dist-packages/tensorboard/compat/proto/histogram_pb2.py", line 34, in <module>
_descriptor.FieldDescriptor(
File "/home/ubuntu/.local/lib/python3.9/site-packages/google/protobuf/descriptor.py", line 561, in __new__
_message.Message._CheckCalledFromGeneratedFile()
TypeError: Descriptors cannot not be created directly.
If this call came from a _pb2.py file, your generated code is out of date and must be regenerated with protoc >= 3.19.0.
If you cannot immediately regenerate your protos, some other possible workarounds are:
1. Downgrade the protobuf package to 3.20.x or lower.
2. Set PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python (but this will use pure-Python parsing and will be much slower).
More information: https://developers.google.com/protocol-buffers/docs/news/2022-05-06#python-updates
In [22]:
!pwd
/home/ubuntu/axolotl
In [24]:
!pip3 install -e . # change depend on needs
!pip3 install protobuf==3.20.3
!pip3 install -U requests
!pip3 install -U --ignore-installed psutil
!pip3 install -U scipy
!pip3 install git+https://github.com/huggingface/peft.git # not for gptq
Defaulting to user installation because normal site-packages is not writeable
Obtaining file:///home/ubuntu/axolotl
Preparing metadata (setup.py) ... done
Collecting transformers@ git+https://github.com/huggingface/transformers.git (from axolotl==0.1)
Cloning https://github.com/huggingface/transformers.git to /tmp/pip-install-48bo5v0h/transformers_137884897c0d4fd8ba70023bd1155185
Running command git clone --filter=blob:none --quiet https://github.com/huggingface/transformers.git /tmp/pip-install-48bo5v0h/transformers_137884897c0d4fd8ba70023bd1155185
Resolved https://github.com/huggingface/transformers.git to commit 539e2281cd97c35ef4122757f26c88f44115fa94
Installing build dependencies ... done
Getting requirements to build wheel ... done
Preparing metadata (pyproject.toml) ... done
Requirement already satisfied: PyYAML==6.0 in /home/ubuntu/.local/lib/python3.9/site-packages (from axolotl==0.1) (6.0)
Requirement already satisfied: accelerate in /home/ubuntu/.local/lib/python3.9/site-packages (from axolotl==0.1) (0.19.0)
Requirement already satisfied: addict in /home/ubuntu/.local/lib/python3.9/site-packages (from axolotl==0.1) (2.4.0)
Requirement already satisfied: bert-score==0.3.13 in /home/ubuntu/.local/lib/python3.9/site-packages (from axolotl==0.1) (0.3.13)
Requirement already satisfied: bitsandbytes>=0.39.0 in /home/ubuntu/.local/lib/python3.9/site-packages (from axolotl==0.1) (0.39.0)
Requirement already satisfied: datasets in /home/ubuntu/.local/lib/python3.9/site-packages (from axolotl==0.1) (2.12.0)
Requirement already satisfied: einops in /home/ubuntu/.local/lib/python3.9/site-packages (from axolotl==0.1) (0.6.1)
Requirement already satisfied: evaluate==0.4.0 in /home/ubuntu/.local/lib/python3.9/site-packages (from axolotl==0.1) (0.4.0)
Requirement already satisfied: fire in /home/ubuntu/.local/lib/python3.9/site-packages (from axolotl==0.1) (0.5.0)
Requirement already satisfied: rouge-score==0.1.2 in /home/ubuntu/.local/lib/python3.9/site-packages (from axolotl==0.1) (0.1.2)
Requirement already satisfied: scikit-learn==1.2.2 in /home/ubuntu/.local/lib/python3.9/site-packages (from axolotl==0.1) (1.2.2)
Requirement already satisfied: scipy in /usr/lib/python3/dist-packages (from axolotl==0.1) (1.3.3)
Requirement already satisfied: sentencepiece in /home/ubuntu/.local/lib/python3.9/site-packages (from axolotl==0.1) (0.1.99)
Requirement already satisfied: wandb in /home/ubuntu/.local/lib/python3.9/site-packages (from axolotl==0.1) (0.15.3)
Requirement already satisfied: xformers in /home/ubuntu/.local/lib/python3.9/site-packages (from axolotl==0.1) (0.0.20)
Requirement already satisfied: torch>=1.0.0 in /home/ubuntu/.local/lib/python3.9/site-packages (from bert-score==0.3.13->axolotl==0.1) (2.0.1+cu118)
Requirement already satisfied: pandas>=1.0.1 in /home/ubuntu/.local/lib/python3.9/site-packages (from bert-score==0.3.13->axolotl==0.1) (2.0.2)
Requirement already satisfied: numpy in /home/ubuntu/.local/lib/python3.9/site-packages (from bert-score==0.3.13->axolotl==0.1) (1.24.3)
Requirement already satisfied: requests in /usr/lib/python3/dist-packages (from bert-score==0.3.13->axolotl==0.1) (2.22.0)
Requirement already satisfied: tqdm>=4.31.1 in /home/ubuntu/.local/lib/python3.9/site-packages (from bert-score==0.3.13->axolotl==0.1) (4.65.0)
Requirement already satisfied: matplotlib in /usr/lib/python3/dist-packages (from bert-score==0.3.13->axolotl==0.1) (3.1.2)
Requirement already satisfied: packaging>=20.9 in /home/ubuntu/.local/lib/python3.9/site-packages (from bert-score==0.3.13->axolotl==0.1) (23.1)
Requirement already satisfied: dill in /home/ubuntu/.local/lib/python3.9/site-packages (from evaluate==0.4.0->axolotl==0.1) (0.3.6)
Requirement already satisfied: xxhash in /home/ubuntu/.local/lib/python3.9/site-packages (from evaluate==0.4.0->axolotl==0.1) (3.2.0)
Requirement already satisfied: multiprocess in /home/ubuntu/.local/lib/python3.9/site-packages (from evaluate==0.4.0->axolotl==0.1) (0.70.14)
Requirement already satisfied: fsspec[http]>=2021.05.0 in /home/ubuntu/.local/lib/python3.9/site-packages (from evaluate==0.4.0->axolotl==0.1) (2023.5.0)
Requirement already satisfied: huggingface-hub>=0.7.0 in /home/ubuntu/.local/lib/python3.9/site-packages (from evaluate==0.4.0->axolotl==0.1) (0.15.1)
Requirement already satisfied: responses<0.19 in /home/ubuntu/.local/lib/python3.9/site-packages (from evaluate==0.4.0->axolotl==0.1) (0.18.0)
Requirement already satisfied: absl-py in /usr/lib/python3/dist-packages (from rouge-score==0.1.2->axolotl==0.1) (0.15.0)
Requirement already satisfied: nltk in /home/ubuntu/.local/lib/python3.9/site-packages (from rouge-score==0.1.2->axolotl==0.1) (3.8.1)
Requirement already satisfied: six>=1.14.0 in /usr/lib/python3/dist-packages (from rouge-score==0.1.2->axolotl==0.1) (1.14.0)
Requirement already satisfied: joblib>=1.1.1 in /home/ubuntu/.local/lib/python3.9/site-packages (from scikit-learn==1.2.2->axolotl==0.1) (1.2.0)
Requirement already satisfied: threadpoolctl>=2.0.0 in /home/ubuntu/.local/lib/python3.9/site-packages (from scikit-learn==1.2.2->axolotl==0.1) (3.1.0)
Requirement already satisfied: psutil in /usr/lib/python3/dist-packages (from accelerate->axolotl==0.1) (5.5.1)
Requirement already satisfied: pyarrow>=8.0.0 in /home/ubuntu/.local/lib/python3.9/site-packages (from datasets->axolotl==0.1) (12.0.0)
Requirement already satisfied: aiohttp in /home/ubuntu/.local/lib/python3.9/site-packages (from datasets->axolotl==0.1) (3.8.4)
Requirement already satisfied: filelock in /usr/lib/python3/dist-packages (from transformers@ git+https://github.com/huggingface/transformers.git->axolotl==0.1) (3.0.12)
Requirement already satisfied: regex!=2019.12.17 in /home/ubuntu/.local/lib/python3.9/site-packages (from transformers@ git+https://github.com/huggingface/transformers.git->axolotl==0.1) (2023.5.5)
Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /home/ubuntu/.local/lib/python3.9/site-packages (from transformers@ git+https://github.com/huggingface/transformers.git->axolotl==0.1) (0.13.3)
Requirement already satisfied: safetensors>=0.3.1 in /home/ubuntu/.local/lib/python3.9/site-packages (from transformers@ git+https://github.com/huggingface/transformers.git->axolotl==0.1) (0.3.1)
Requirement already satisfied: termcolor in /usr/lib/python3/dist-packages (from fire->axolotl==0.1) (1.1.0)
Requirement already satisfied: Click!=8.0.0,>=7.0 in /usr/lib/python3/dist-packages (from wandb->axolotl==0.1) (7.0)
Requirement already satisfied: GitPython!=3.1.29,>=1.0.0 in /home/ubuntu/.local/lib/python3.9/site-packages (from wandb->axolotl==0.1) (3.1.31)
Requirement already satisfied: sentry-sdk>=1.0.0 in /home/ubuntu/.local/lib/python3.9/site-packages (from wandb->axolotl==0.1) (1.25.0)
Requirement already satisfied: docker-pycreds>=0.4.0 in /home/ubuntu/.local/lib/python3.9/site-packages (from wandb->axolotl==0.1) (0.4.0)
Requirement already satisfied: pathtools in /home/ubuntu/.local/lib/python3.9/site-packages (from wandb->axolotl==0.1) (0.1.2)
Requirement already satisfied: setproctitle in /home/ubuntu/.local/lib/python3.9/site-packages (from wandb->axolotl==0.1) (1.3.2)
Requirement already satisfied: setuptools in /usr/lib/python3/dist-packages (from wandb->axolotl==0.1) (45.2.0)
Requirement already satisfied: appdirs>=1.4.3 in /usr/lib/python3/dist-packages (from wandb->axolotl==0.1) (1.4.3)
Requirement already satisfied: typing-extensions in /usr/lib/python3/dist-packages (from wandb->axolotl==0.1) (3.10.0.2)
Requirement already satisfied: protobuf!=4.21.0,<5,>=3.15.0 in /home/ubuntu/.local/lib/python3.9/site-packages (from wandb->axolotl==0.1) (4.23.2)
Requirement already satisfied: pyre-extensions==0.0.29 in /home/ubuntu/.local/lib/python3.9/site-packages (from xformers->axolotl==0.1) (0.0.29)
Requirement already satisfied: typing-inspect in /home/ubuntu/.local/lib/python3.9/site-packages (from pyre-extensions==0.0.29->xformers->axolotl==0.1) (0.9.0)
Requirement already satisfied: sympy in /home/ubuntu/.local/lib/python3.9/site-packages (from torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (1.11.1)
Requirement already satisfied: networkx in /usr/lib/python3/dist-packages (from torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (2.4)
Requirement already satisfied: jinja2 in /usr/lib/python3/dist-packages (from torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (2.10.1)
Requirement already satisfied: triton==2.0.0 in /home/ubuntu/.local/lib/python3.9/site-packages (from torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (2.0.0)
Requirement already satisfied: cmake in /home/ubuntu/.local/lib/python3.9/site-packages (from triton==2.0.0->torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (3.25.0)
Requirement already satisfied: lit in /home/ubuntu/.local/lib/python3.9/site-packages (from triton==2.0.0->torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (15.0.7)
Requirement already satisfied: attrs>=17.3.0 in /usr/lib/python3/dist-packages (from aiohttp->datasets->axolotl==0.1) (19.3.0)
Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /home/ubuntu/.local/lib/python3.9/site-packages (from aiohttp->datasets->axolotl==0.1) (3.1.0)
Requirement already satisfied: multidict<7.0,>=4.5 in /home/ubuntu/.local/lib/python3.9/site-packages (from aiohttp->datasets->axolotl==0.1) (6.0.4)
Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /home/ubuntu/.local/lib/python3.9/site-packages (from aiohttp->datasets->axolotl==0.1) (4.0.2)
Requirement already satisfied: yarl<2.0,>=1.0 in /home/ubuntu/.local/lib/python3.9/site-packages (from aiohttp->datasets->axolotl==0.1) (1.9.2)
Requirement already satisfied: frozenlist>=1.1.1 in /home/ubuntu/.local/lib/python3.9/site-packages (from aiohttp->datasets->axolotl==0.1) (1.3.3)
Requirement already satisfied: aiosignal>=1.1.2 in /home/ubuntu/.local/lib/python3.9/site-packages (from aiohttp->datasets->axolotl==0.1) (1.3.1)
Requirement already satisfied: gitdb<5,>=4.0.1 in /home/ubuntu/.local/lib/python3.9/site-packages (from GitPython!=3.1.29,>=1.0.0->wandb->axolotl==0.1) (4.0.10)
Requirement already satisfied: python-dateutil>=2.8.2 in /home/ubuntu/.local/lib/python3.9/site-packages (from pandas>=1.0.1->bert-score==0.3.13->axolotl==0.1) (2.8.2)
Requirement already satisfied: pytz>=2020.1 in /home/ubuntu/.local/lib/python3.9/site-packages (from pandas>=1.0.1->bert-score==0.3.13->axolotl==0.1) (2023.3)
Requirement already satisfied: tzdata>=2022.1 in /home/ubuntu/.local/lib/python3.9/site-packages (from pandas>=1.0.1->bert-score==0.3.13->axolotl==0.1) (2023.3)
Requirement already satisfied: urllib3>=1.25.10 in /home/ubuntu/.local/lib/python3.9/site-packages (from responses<0.19->evaluate==0.4.0->axolotl==0.1) (2.0.2)
Requirement already satisfied: certifi in /usr/lib/python3/dist-packages (from sentry-sdk>=1.0.0->wandb->axolotl==0.1) (2019.11.28)
Requirement already satisfied: smmap<6,>=3.0.1 in /home/ubuntu/.local/lib/python3.9/site-packages (from gitdb<5,>=4.0.1->GitPython!=3.1.29,>=1.0.0->wandb->axolotl==0.1) (5.0.0)
Requirement already satisfied: idna>=2.0 in /usr/lib/python3/dist-packages (from yarl<2.0,>=1.0->aiohttp->datasets->axolotl==0.1) (2.8)
Requirement already satisfied: mpmath>=0.19 in /home/ubuntu/.local/lib/python3.9/site-packages (from sympy->torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (1.2.1)
Requirement already satisfied: mypy-extensions>=0.3.0 in /home/ubuntu/.local/lib/python3.9/site-packages (from typing-inspect->pyre-extensions==0.0.29->xformers->axolotl==0.1) (1.0.0)
Installing collected packages: axolotl
Attempting uninstall: axolotl
Found existing installation: axolotl 0.1
Uninstalling axolotl-0.1:
Successfully uninstalled axolotl-0.1
Running setup.py develop for axolotl
Successfully installed axolotl
Defaulting to user installation because normal site-packages is not writeable
Collecting protobuf==3.20.3
Downloading protobuf-3.20.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl (1.0 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.0/1.0 MB 110.7 MB/s eta 0:00:00
Installing collected packages: protobuf
Attempting uninstall: protobuf
Found existing installation: protobuf 4.23.2
Uninstalling protobuf-4.23.2:
Successfully uninstalled protobuf-4.23.2
Successfully installed protobuf-3.20.3
Defaulting to user installation because normal site-packages is not writeable
Requirement already satisfied: requests in /usr/lib/python3/dist-packages (2.22.0)
Collecting requests
Downloading requests-2.31.0-py3-none-any.whl (62 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 62.6/62.6 kB 18.1 MB/s eta 0:00:00
Requirement already satisfied: charset-normalizer<4,>=2 in /home/ubuntu/.local/lib/python3.9/site-packages (from requests) (3.1.0)
Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests) (2.8)
Requirement already satisfied: urllib3<3,>=1.21.1 in /home/ubuntu/.local/lib/python3.9/site-packages (from requests) (2.0.2)
Requirement already satisfied: certifi>=2017.4.17 in /usr/lib/python3/dist-packages (from requests) (2019.11.28)
Installing collected packages: requests
Successfully installed requests-2.31.0
Defaulting to user installation because normal site-packages is not writeable
Collecting psutil
Downloading psutil-5.9.5-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (282 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 282.1/282.1 kB 46.8 MB/s eta 0:00:00
Installing collected packages: psutil
Successfully installed psutil-5.9.5
Defaulting to user installation because normal site-packages is not writeable
Requirement already satisfied: scipy in /usr/lib/python3/dist-packages (1.3.3)
Collecting scipy
Downloading scipy-1.10.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (34.5 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 34.5/34.5 MB 87.6 MB/s eta 0:00:0000:0100:01
Requirement already satisfied: numpy<1.27.0,>=1.19.5 in /home/ubuntu/.local/lib/python3.9/site-packages (from scipy) (1.24.3)
Installing collected packages: scipy
Successfully installed scipy-1.10.1
Defaulting to user installation because normal site-packages is not writeable
Collecting git+https://github.com/huggingface/peft.git
Cloning https://github.com/huggingface/peft.git to /tmp/pip-req-build-3755vabw
Running command git clone --filter=blob:none --quiet https://github.com/huggingface/peft.git /tmp/pip-req-build-3755vabw
Resolved https://github.com/huggingface/peft.git to commit fcff23f005fc7bfb816ad1f55360442c170cd5f5
Installing build dependencies ... done
Getting requirements to build wheel ... done
Preparing metadata (pyproject.toml) ... done
Requirement already satisfied: numpy>=1.17 in /home/ubuntu/.local/lib/python3.9/site-packages (from peft==0.4.0.dev0) (1.24.3)
Requirement already satisfied: packaging>=20.0 in /home/ubuntu/.local/lib/python3.9/site-packages (from peft==0.4.0.dev0) (23.1)
Requirement already satisfied: psutil in /home/ubuntu/.local/lib/python3.9/site-packages (from peft==0.4.0.dev0) (5.9.5)
Requirement already satisfied: pyyaml in /home/ubuntu/.local/lib/python3.9/site-packages (from peft==0.4.0.dev0) (6.0)
Requirement already satisfied: torch>=1.13.0 in /home/ubuntu/.local/lib/python3.9/site-packages (from peft==0.4.0.dev0) (2.0.1+cu118)
Requirement already satisfied: transformers in /home/ubuntu/.local/lib/python3.9/site-packages (from peft==0.4.0.dev0) (4.30.0.dev0)
Requirement already satisfied: accelerate in /home/ubuntu/.local/lib/python3.9/site-packages (from peft==0.4.0.dev0) (0.19.0)
Requirement already satisfied: filelock in /usr/lib/python3/dist-packages (from torch>=1.13.0->peft==0.4.0.dev0) (3.0.12)
Requirement already satisfied: typing-extensions in /usr/lib/python3/dist-packages (from torch>=1.13.0->peft==0.4.0.dev0) (3.10.0.2)
Requirement already satisfied: sympy in /home/ubuntu/.local/lib/python3.9/site-packages (from torch>=1.13.0->peft==0.4.0.dev0) (1.11.1)
Requirement already satisfied: networkx in /usr/lib/python3/dist-packages (from torch>=1.13.0->peft==0.4.0.dev0) (2.4)
Requirement already satisfied: jinja2 in /usr/lib/python3/dist-packages (from torch>=1.13.0->peft==0.4.0.dev0) (2.10.1)
Requirement already satisfied: triton==2.0.0 in /home/ubuntu/.local/lib/python3.9/site-packages (from torch>=1.13.0->peft==0.4.0.dev0) (2.0.0)
Requirement already satisfied: cmake in /home/ubuntu/.local/lib/python3.9/site-packages (from triton==2.0.0->torch>=1.13.0->peft==0.4.0.dev0) (3.25.0)
Requirement already satisfied: lit in /home/ubuntu/.local/lib/python3.9/site-packages (from triton==2.0.0->torch>=1.13.0->peft==0.4.0.dev0) (15.0.7)
Requirement already satisfied: huggingface-hub<1.0,>=0.14.1 in /home/ubuntu/.local/lib/python3.9/site-packages (from transformers->peft==0.4.0.dev0) (0.15.1)
Requirement already satisfied: regex!=2019.12.17 in /home/ubuntu/.local/lib/python3.9/site-packages (from transformers->peft==0.4.0.dev0) (2023.5.5)
Requirement already satisfied: requests in /home/ubuntu/.local/lib/python3.9/site-packages (from transformers->peft==0.4.0.dev0) (2.31.0)
Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /home/ubuntu/.local/lib/python3.9/site-packages (from transformers->peft==0.4.0.dev0) (0.13.3)
Requirement already satisfied: safetensors>=0.3.1 in /home/ubuntu/.local/lib/python3.9/site-packages (from transformers->peft==0.4.0.dev0) (0.3.1)
Requirement already satisfied: tqdm>=4.27 in /home/ubuntu/.local/lib/python3.9/site-packages (from transformers->peft==0.4.0.dev0) (4.65.0)
Requirement already satisfied: fsspec in /home/ubuntu/.local/lib/python3.9/site-packages (from huggingface-hub<1.0,>=0.14.1->transformers->peft==0.4.0.dev0) (2023.5.0)
Requirement already satisfied: charset-normalizer<4,>=2 in /home/ubuntu/.local/lib/python3.9/site-packages (from requests->transformers->peft==0.4.0.dev0) (3.1.0)
Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests->transformers->peft==0.4.0.dev0) (2.8)
Requirement already satisfied: urllib3<3,>=1.21.1 in /home/ubuntu/.local/lib/python3.9/site-packages (from requests->transformers->peft==0.4.0.dev0) (2.0.2)
Requirement already satisfied: certifi>=2017.4.17 in /usr/lib/python3/dist-packages (from requests->transformers->peft==0.4.0.dev0) (2019.11.28)
Requirement already satisfied: mpmath>=0.19 in /home/ubuntu/.local/lib/python3.9/site-packages (from sympy->torch>=1.13.0->peft==0.4.0.dev0) (1.2.1)
Building wheels for collected packages: peft
Building wheel for peft (pyproject.toml) ... done
Created wheel for peft: filename=peft-0.4.0.dev0-py3-none-any.whl size=57784 sha256=537d985f2ccbf80c6a275d4257085a4225af3e6ddcbcde6bece95cb9c3f2f925
Stored in directory: /tmp/pip-ephem-wheel-cache-zzasgnxp/wheels/2d/60/1b/0edd9dc0f0c489738b1166bc1b0b560ee368f7721f89d06e3a
Successfully built peft
Installing collected packages: peft
Successfully installed peft-0.4.0.dev0
In [25]:
!echo $LD_LIBRARY_PATH
In [26]:
%env LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu
env: LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu
In [29]:
!accelerate config default
accelerate configuration saved at /home/ubuntu/.cache/huggingface/accelerate/default_config.yaml
In [31]:
!wget https://raw.githubusercontent.com/utensil/axolotl/falcon-7b-qlora/examples/falcon/config-7b-qlora.yml
--2023-06-03 15:51:47-- https://raw.githubusercontent.com/utensil/axolotl/falcon-7b-qlora/examples/falcon/config-7b-qlora.yml Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.111.133, 185.199.109.133, 185.199.108.133, ... Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.111.133|:443... connected. HTTP request sent, awaiting response... 200 OK Length: 2238 (2.2K) [text/plain] Saving to: ‘config-7b-qlora.yml’ config-7b-qlora.yml 100%[===================>] 2.19K --.-KB/s in 0s 2023-06-03 15:51:47 (102 MB/s) - ‘config-7b-qlora.yml’ saved [2238/2238]
In [33]:
!cp config-7b-qlora.yml config-1b-qlora.yml
!sed -i -e 's/falcon-7b/falcon-rw-1b/g' -e 's/wandb_project: falcon-qlora/wandb_project: /g' -e 's/bf16: true/bf16: false/g' -e 's/tf32: true/tf32: false/g' config-1b-qlora.yml
In [34]:
!cat config-1b-qlora.yml
# 1b: tiiuae/falcon-rw-1b
# 40b: tiiuae/falcon-40b
base_model: tiiuae/falcon-rw-1b
base_model_config: tiiuae/falcon-rw-1b
# required by falcon custom model code: https://huggingface.co/tiiuae/falcon-rw-1b/tree/main
trust_remote_code: true
model_type: AutoModelForCausalLM
tokenizer_type: AutoTokenizer
load_in_8bit: false
# enable 4bit for QLoRA
load_in_4bit: true
gptq: false
strict: false
push_dataset_to_hub:
datasets:
- path: QingyiSi/Alpaca-CoT
data_files:
- Chain-of-Thought/formatted_cot_data/gsm8k_train.json
type: "alpaca:chat"
dataset_prepared_path: last_run_prepared
val_set_size: 0.01
# enable QLoRA
adapter: qlora
lora_model_dir:
sequence_len: 2048
max_packed_sequence_len:
# hyperparameters from QLoRA paper Appendix B.2
# "We find hyperparameters to be largely robust across datasets"
lora_r: 64
lora_alpha: 16
# 0.1 for models up to 13B
# 0.05 for 33B and 65B models
lora_dropout: 0.05
# add LoRA modules on all linear layers of the base model
lora_target_modules:
lora_target_linear: true
lora_fan_in_fan_out:
wandb_project:
wandb_watch:
wandb_run_id:
wandb_log_model:
output_dir: ./qlora-out
# QLoRA paper Table 9
# - 16 for 7b & 13b
# - 32 for 33b, 64 for 64b
# Max size tested on A6000
# - 7b: 40
# - 40b: 4
# decrease if OOM, increase for max VRAM utilization
micro_batch_size: 1
gradient_accumulation_steps: 2
num_epochs: 3
# Optimizer for QLoRA
optimizer: paged_adamw_32bit
torchdistx_path:
lr_scheduler: cosine
# QLoRA paper Table 9
# - 2e-4 for 7b & 13b
# - 1e-4 for 33b & 64b
learning_rate: 0.0002
train_on_inputs: false
group_by_length: false
bf16: false
fp16: false
tf32: false
gradient_checkpointing: true
# stop training after this many evaluation losses have increased in a row
# https://huggingface.co/transformers/v4.2.2/_modules/transformers/trainer_callback.html#EarlyStoppingCallback
early_stopping_patience: 3
resume_from_checkpoint:
auto_resume_from_checkpoints: true
local_rank:
logging_steps: 1
xformers_attention: true
flash_attention:
gptq_groupsize:
gptq_model_v1:
warmup_steps: 10
eval_steps: 5
save_steps: 10
debug:
deepspeed:
weight_decay: 0.000001
fsdp:
fsdp_config:
special_tokens:
pad_token: "<|endoftext|>"
bos_token: ">>ABSTRACT<<"
eos_token: "<|endoftext|>"
In [35]:
%env WANDB_MODE=offline
env: WANDB_MODE=offline
In [36]:
!accelerate launch scripts/finetune.py config-1b-qlora.yml
===================================BUG REPORT===================================
Welcome to bitsandbytes. For bug reports, please run
python -m bitsandbytes
and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues
================================================================================
bin /home/ubuntu/.local/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda118.so
CUDA SETUP: CUDA runtime path found: /usr/lib/x86_64-linux-gnu/libcudart.so.11.0
CUDA SETUP: Highest compute capability among GPUs detected: 8.6
CUDA SETUP: Detected CUDA version 118
CUDA SETUP: Loading binary /home/ubuntu/.local/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda118.so...
WARNING:root:`trust_remote_code` is set to true. Please make sure that you reviewed the remote code/model.
INFO:root:loading tokenizer... tiiuae/falcon-rw-1b
INFO:filelock:Lock 139620631419392 acquired on /home/ubuntu/.cache/huggingface/hub/models--tiiuae--falcon-rw-1b/blobs/31a4ebf2104713af5922ab75c18e61597a6f076e.lock
Downloading (…)okenizer_config.json: 100%|█████| 255/255 [00:00<00:00, 63.1kB/s]
INFO:filelock:Lock 139620631419392 released on /home/ubuntu/.cache/huggingface/hub/models--tiiuae--falcon-rw-1b/blobs/31a4ebf2104713af5922ab75c18e61597a6f076e.lock
INFO:filelock:Lock 139620631419152 acquired on /home/ubuntu/.cache/huggingface/hub/models--tiiuae--falcon-rw-1b/blobs/84ef7fb594b5c0979e48bdeddb60a0adef33df0b.lock
Downloading (…)olve/main/vocab.json: 100%|███| 798k/798k [00:00<00:00, 3.37MB/s]
INFO:filelock:Lock 139620631419152 released on /home/ubuntu/.cache/huggingface/hub/models--tiiuae--falcon-rw-1b/blobs/84ef7fb594b5c0979e48bdeddb60a0adef33df0b.lock
INFO:filelock:Lock 139620631419392 acquired on /home/ubuntu/.cache/huggingface/hub/models--tiiuae--falcon-rw-1b/blobs/6636bda4a1fd7a63653dffb22683b8162c8de956.lock
Downloading (…)olve/main/merges.txt: 100%|███| 456k/456k [00:00<00:00, 2.57MB/s]
INFO:filelock:Lock 139620631419392 released on /home/ubuntu/.cache/huggingface/hub/models--tiiuae--falcon-rw-1b/blobs/6636bda4a1fd7a63653dffb22683b8162c8de956.lock
INFO:filelock:Lock 139620631419152 acquired on /home/ubuntu/.cache/huggingface/hub/models--tiiuae--falcon-rw-1b/blobs/6dc5e94ca994de07d374e33698489de9cff48a02.lock
Downloading (…)/main/tokenizer.json: 100%|█| 2.11M/2.11M [00:00<00:00, 11.6MB/s]
INFO:filelock:Lock 139620631419152 released on /home/ubuntu/.cache/huggingface/hub/models--tiiuae--falcon-rw-1b/blobs/6dc5e94ca994de07d374e33698489de9cff48a02.lock
INFO:filelock:Lock 139620631419392 acquired on /home/ubuntu/.cache/huggingface/hub/models--tiiuae--falcon-rw-1b/blobs/0204ed10c186a4c7c68f55dff8f26087a45898d6.lock
Downloading (…)cial_tokens_map.json: 100%|████| 99.0/99.0 [00:00<00:00, 146kB/s]
INFO:filelock:Lock 139620631419392 released on /home/ubuntu/.cache/huggingface/hub/models--tiiuae--falcon-rw-1b/blobs/0204ed10c186a4c7c68f55dff8f26087a45898d6.lock
Using pad_token, but it is not set yet.
INFO:root:Unable to find prepared dataset in last_run_prepared/0ecc5b78e3ce4254b22e749b093712b4
INFO:root:Loading raw datasets...
Downloading readme: 100%|██████████████████| 8.26k/8.26k [00:00<00:00, 31.1MB/s]
Downloading and preparing dataset json/QingyiSi--Alpaca-CoT to /home/ubuntu/.cache/huggingface/datasets/QingyiSi___json/QingyiSi--Alpaca-CoT-2953efcfeb19f105/0.0.0/e347ab1c932092252e717ff3f949105a4dd28b27e842dd53157d2f72e276c2e4...
Downloading data files: 0%| | 0/1 [00:00<?, ?it/s]
Downloading data: 100%|█████████████████████| 4.45M/4.45M [00:00<00:00, 116MB/s]
Downloading data files: 100%|█████████████████████| 1/1 [00:00<00:00, 3.85it/s]
Extracting data files: 100%|████████████████████| 1/1 [00:00<00:00, 2560.63it/s]
Dataset json downloaded and prepared to /home/ubuntu/.cache/huggingface/datasets/QingyiSi___json/QingyiSi--Alpaca-CoT-2953efcfeb19f105/0.0.0/e347ab1c932092252e717ff3f949105a4dd28b27e842dd53157d2f72e276c2e4. Subsequent calls will reuse this data.
100%|████████████████████████████████████████████| 1/1 [00:00<00:00, 983.89it/s]
INFO:root:tokenizing, merging, and shuffling master dataset
INFO:root:Saving merged prepared dataset to disk... last_run_prepared/0ecc5b78e3ce4254b22e749b093712b4
INFO:root:loading model and peft_config...
INFO:filelock:Lock 139620622432576 acquired on /home/ubuntu/.cache/huggingface/hub/models--tiiuae--falcon-rw-1b/blobs/c6e815d7380adbb2b091fd7bc7580d72dee0bc03.lock
Downloading (…)lve/main/config.json: 100%|██████| 665/665 [00:00<00:00, 788kB/s]
INFO:filelock:Lock 139620622432576 released on /home/ubuntu/.cache/huggingface/hub/models--tiiuae--falcon-rw-1b/blobs/c6e815d7380adbb2b091fd7bc7580d72dee0bc03.lock
INFO:filelock:Lock 139621809190704 acquired on /home/ubuntu/.cache/huggingface/hub/models--tiiuae--falcon-rw-1b/blobs/ab845c0d34462b3b3391d3b8119c11816e9c02ce.lock
Downloading (…)/configuration_RW.py: 100%|█| 2.61k/2.61k [00:00<00:00, 3.72MB/s]
INFO:filelock:Lock 139621809190704 released on /home/ubuntu/.cache/huggingface/hub/models--tiiuae--falcon-rw-1b/blobs/ab845c0d34462b3b3391d3b8119c11816e9c02ce.lock
A new version of the following files was downloaded from https://huggingface.co/tiiuae/falcon-rw-1b:
- configuration_RW.py
. Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.
INFO:filelock:Lock 139625672846544 acquired on /home/ubuntu/.cache/huggingface/hub/models--tiiuae--falcon-rw-1b/blobs/3e4e76c47352197b871178c8c508bc8c5942a2f8.lock
Downloading (…)main/modelling_RW.py: 100%|█| 47.5k/47.5k [00:00<00:00, 67.3MB/s]
INFO:filelock:Lock 139625672846544 released on /home/ubuntu/.cache/huggingface/hub/models--tiiuae--falcon-rw-1b/blobs/3e4e76c47352197b871178c8c508bc8c5942a2f8.lock
A new version of the following files was downloaded from https://huggingface.co/tiiuae/falcon-rw-1b:
- modelling_RW.py
. Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.
INFO:filelock:Lock 139621809986576 acquired on /home/ubuntu/.cache/huggingface/hub/models--tiiuae--falcon-rw-1b/blobs/3a0d68f0309c8f7ec913f51edf8bf2eca849477c9f53db727f49ffa2f6019251.lock
Downloading pytorch_model.bin: 100%|████████| 2.62G/2.62G [00:13<00:00, 188MB/s]
INFO:filelock:Lock 139621809986576 released on /home/ubuntu/.cache/huggingface/hub/models--tiiuae--falcon-rw-1b/blobs/3a0d68f0309c8f7ec913f51edf8bf2eca849477c9f53db727f49ffa2f6019251.lock
INFO:filelock:Lock 139621809191040 acquired on /home/ubuntu/.cache/huggingface/hub/models--tiiuae--falcon-rw-1b/blobs/2cb1ddd9ee691eb96ad0baa19b67447cd7be0275.lock
Downloading (…)neration_config.json: 100%|██████| 111/111 [00:00<00:00, 133kB/s]
INFO:filelock:Lock 139621809191040 released on /home/ubuntu/.cache/huggingface/hub/models--tiiuae--falcon-rw-1b/blobs/2cb1ddd9ee691eb96ad0baa19b67447cd7be0275.lock
INFO:root:converting PEFT model w/ prepare_model_for_int8_training
/home/ubuntu/.local/lib/python3.9/site-packages/peft/utils/other.py:76: FutureWarning: prepare_model_for_int8_training is deprecated and will be removed in a future version. Use prepare_model_for_kbit_training instead.
warnings.warn(
INFO:root:found linear modules: ['dense_4h_to_h', 'query_key_value', 'dense_h_to_4h', 'dense']
trainable params: 50331648 || all params: 757911552 || trainable%: 6.6408339953630895
INFO:root:Compiling torch model
INFO:root:Pre-saving adapter config to ./qlora-out
INFO:root:Starting trainer...
huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
To disable this warning, you can either:
- Avoid using `tokenizers` before the fork if possible
- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
To disable this warning, you can either:
- Avoid using `tokenizers` before the fork if possible
- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
To disable this warning, you can either:
- Avoid using `tokenizers` before the fork if possible
- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
To disable this warning, you can either:
- Avoid using `tokenizers` before the fork if possible
- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
Traceback (most recent call last):
File "/usr/lib/python3.9/runpy.py", line 197, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/usr/lib/python3.9/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "/home/ubuntu/.local/lib/python3.9/site-packages/wandb/__main__.py", line 1, in <module>
from wandb.cli import cli
File "/home/ubuntu/.local/lib/python3.9/site-packages/wandb/cli/cli.py", line 933, in <module>
def launch_sweep(
File "/usr/lib/python3/dist-packages/click/core.py", line 1234, in decorator
cmd = command(*args, **kwargs)(f)
File "/usr/lib/python3/dist-packages/click/decorators.py", line 115, in decorator
cmd = _make_command(f, name, attrs, cls)
File "/usr/lib/python3/dist-packages/click/decorators.py", line 88, in _make_command
return cls(name=name or f.__name__.lower().replace('_', '-'),
TypeError: __init__() got an unexpected keyword argument 'no_args_is_help'
Traceback (most recent call last):
File "/home/ubuntu/axolotl/scripts/finetune.py", line 295, in <module>
fire.Fire(train)
File "/home/ubuntu/.local/lib/python3.9/site-packages/fire/core.py", line 141, in Fire
component_trace = _Fire(component, args, parsed_flag_args, context, name)
File "/home/ubuntu/.local/lib/python3.9/site-packages/fire/core.py", line 475, in _Fire
component, remaining_args = _CallAndUpdateTrace(
File "/home/ubuntu/.local/lib/python3.9/site-packages/fire/core.py", line 691, in _CallAndUpdateTrace
component = fn(*varargs, **kwargs)
File "/home/ubuntu/axolotl/scripts/finetune.py", line 282, in train
trainer.train(resume_from_checkpoint=resume_from_checkpoint)
File "/home/ubuntu/.local/lib/python3.9/site-packages/transformers/trainer.py", line 1661, in train
return inner_training_loop(
File "/home/ubuntu/.local/lib/python3.9/site-packages/transformers/trainer.py", line 1870, in _inner_training_loop
self.control = self.callback_handler.on_train_begin(args, self.state, self.control)
File "/home/ubuntu/.local/lib/python3.9/site-packages/transformers/trainer_callback.py", line 353, in on_train_begin
return self.call_event("on_train_begin", args, state, control)
File "/home/ubuntu/.local/lib/python3.9/site-packages/transformers/trainer_callback.py", line 397, in call_event
result = getattr(callback, event)(
File "/home/ubuntu/.local/lib/python3.9/site-packages/transformers/integrations.py", line 774, in on_train_begin
self.setup(args, state, model, **kwargs)
File "/home/ubuntu/.local/lib/python3.9/site-packages/transformers/integrations.py", line 748, in setup
self._wandb.init(
File "/home/ubuntu/.local/lib/python3.9/site-packages/wandb/sdk/wandb_init.py", line 1169, in init
raise e
File "/home/ubuntu/.local/lib/python3.9/site-packages/wandb/sdk/wandb_init.py", line 1146, in init
wi.setup(kwargs)
File "/home/ubuntu/.local/lib/python3.9/site-packages/wandb/sdk/wandb_init.py", line 172, in setup
self._wl = wandb_setup.setup(settings=setup_settings)
File "/home/ubuntu/.local/lib/python3.9/site-packages/wandb/sdk/wandb_setup.py", line 327, in setup
ret = _setup(settings=settings)
File "/home/ubuntu/.local/lib/python3.9/site-packages/wandb/sdk/wandb_setup.py", line 320, in _setup
wl = _WandbSetup(settings=settings)
File "/home/ubuntu/.local/lib/python3.9/site-packages/wandb/sdk/wandb_setup.py", line 303, in __init__
_WandbSetup._instance = _WandbSetup__WandbSetup(settings=settings, pid=pid)
File "/home/ubuntu/.local/lib/python3.9/site-packages/wandb/sdk/wandb_setup.py", line 114, in __init__
self._setup()
File "/home/ubuntu/.local/lib/python3.9/site-packages/wandb/sdk/wandb_setup.py", line 250, in _setup
self._setup_manager()
File "/home/ubuntu/.local/lib/python3.9/site-packages/wandb/sdk/wandb_setup.py", line 277, in _setup_manager
self._manager = wandb_manager._Manager(settings=self._settings)
File "/home/ubuntu/.local/lib/python3.9/site-packages/wandb/sdk/wandb_manager.py", line 145, in __init__
self._service.start()
File "/home/ubuntu/.local/lib/python3.9/site-packages/wandb/sdk/service/service.py", line 199, in start
self._launch_server()
File "/home/ubuntu/.local/lib/python3.9/site-packages/wandb/sdk/service/service.py", line 193, in _launch_server
_sentry.reraise(e)
File "/home/ubuntu/.local/lib/python3.9/site-packages/wandb/analytics/sentry.py", line 146, in reraise
raise exc.with_traceback(sys.exc_info()[2])
File "/home/ubuntu/.local/lib/python3.9/site-packages/wandb/sdk/service/service.py", line 191, in _launch_server
self._wait_for_ports(fname, proc=internal_proc)
File "/home/ubuntu/.local/lib/python3.9/site-packages/wandb/sdk/service/service.py", line 116, in _wait_for_ports
raise ServiceStartProcessError(
wandb.sdk.service.service.ServiceStartProcessError: The wandb service process exited with 1. Ensure that `sys.executable` is a valid python interpreter. You can override it with the `_executable` setting or with the `WANDB__EXECUTABLE` environment variable.
Traceback (most recent call last):
File "/home/ubuntu/.local/bin/accelerate", line 8, in <module>
sys.exit(main())
File "/home/ubuntu/.local/lib/python3.9/site-packages/accelerate/commands/accelerate_cli.py", line 45, in main
args.func(args)
File "/home/ubuntu/.local/lib/python3.9/site-packages/accelerate/commands/launch.py", line 918, in launch_command
simple_launcher(args)
File "/home/ubuntu/.local/lib/python3.9/site-packages/accelerate/commands/launch.py", line 580, in simple_launcher
raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd)
subprocess.CalledProcessError: Command '['/usr/bin/python', 'scripts/finetune.py', 'config-1b-qlora.yml']' returned non-zero exit status 1.
In [37]:
# https://github.com/explosion/spaCy/issues/7160
!pip3 install click --upgrade
Defaulting to user installation because normal site-packages is not writeable
Requirement already satisfied: click in /usr/lib/python3/dist-packages (7.0)
Collecting click
Downloading click-8.1.3-py3-none-any.whl (96 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 96.6/96.6 kB 29.5 MB/s eta 0:00:00
Installing collected packages: click
Successfully installed click-8.1.3
In [38]:
!accelerate launch scripts/finetune.py config-1b-qlora.yml
===================================BUG REPORT=================================== Welcome to bitsandbytes. For bug reports, please run python -m bitsandbytes and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues ================================================================================ bin /home/ubuntu/.local/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda118.so CUDA SETUP: CUDA runtime path found: /usr/lib/x86_64-linux-gnu/libcudart.so.11.0 CUDA SETUP: Highest compute capability among GPUs detected: 8.6 CUDA SETUP: Detected CUDA version 118 CUDA SETUP: Loading binary /home/ubuntu/.local/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda118.so... WARNING:root:`trust_remote_code` is set to true. Please make sure that you reviewed the remote code/model. INFO:root:loading tokenizer... tiiuae/falcon-rw-1b Using pad_token, but it is not set yet. INFO:root:Loading prepared dataset from disk at last_run_prepared/0ecc5b78e3ce4254b22e749b093712b4... INFO:root:Prepared dataset loaded from disk... INFO:root:loading model and peft_config... INFO:root:converting PEFT model w/ prepare_model_for_int8_training /home/ubuntu/.local/lib/python3.9/site-packages/peft/utils/other.py:76: FutureWarning: prepare_model_for_int8_training is deprecated and will be removed in a future version. Use prepare_model_for_kbit_training instead. warnings.warn( INFO:root:found linear modules: ['dense_h_to_4h', 'dense_4h_to_h', 'dense', 'query_key_value'] trainable params: 50331648 || all params: 757911552 || trainable%: 6.6408339953630895 INFO:root:Compiling torch model INFO:root:Pre-saving adapter config to ./qlora-out INFO:root:Starting trainer... wandb: Tracking run with wandb version 0.15.3 wandb: W&B syncing is set to `offline` in this directory. wandb: Run `wandb online` or set WANDB_MODE=online to enable cloud syncing. 0%| | 0/11097 [00:00<?, ?it/s]You're using a GPT2TokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding. {'loss': 1.858, 'learning_rate': 2e-05, 'epoch': 0.0} {'loss': 1.6529, 'learning_rate': 4e-05, 'epoch': 0.0} {'loss': 1.642, 'learning_rate': 6e-05, 'epoch': 0.0} {'loss': 1.7997, 'learning_rate': 8e-05, 'epoch': 0.0} {'loss': 1.7844, 'learning_rate': 0.0001, 'epoch': 0.0} 0%| | 5/11097 [00:03<2:01:58, 1.52it/s] 0%| | 0/75 [00:00<?, ?it/s] 4%|█▊ | 3/75 [00:00<00:03, 22.01it/s] 8%|███▌ | 6/75 [00:00<00:04, 14.86it/s] 11%|████▋ | 8/75 [00:00<00:05, 12.59it/s] 13%|█████▋ | 10/75 [00:00<00:05, 12.74it/s] 16%|██████▉ | 12/75 [00:00<00:04, 12.87it/s] 19%|████████ | 14/75 [00:01<00:04, 12.49it/s] 21%|█████████▏ | 16/75 [00:01<00:04, 12.39it/s] 24%|██████████▎ | 18/75 [00:01<00:04, 12.63it/s] 27%|███████████▍ | 20/75 [00:01<00:04, 12.07it/s] 29%|████████████▌ | 22/75 [00:01<00:04, 11.96it/s] 32%|█████████████▊ | 24/75 [00:01<00:04, 11.58it/s] 35%|██████████████▉ | 26/75 [00:02<00:04, 12.23it/s] 37%|████████████████ | 28/75 [00:02<00:03, 12.15it/s] 40%|█████████████████▏ | 30/75 [00:02<00:03, 11.76it/s] 43%|██████████████████▎ | 32/75 [00:02<00:03, 11.74it/s] 45%|███████████████████▍ | 34/75 [00:02<00:03, 11.67it/s] 48%|████████████████████▋ | 36/75 [00:02<00:03, 11.35it/s] 51%|█████████████████████▊ | 38/75 [00:03<00:03, 11.50it/s] 53%|██████████████████████▉ | 40/75 [00:03<00:03, 11.52it/s] 56%|████████████████████████ | 42/75 [00:03<00:02, 11.56it/s] 59%|█████████████████████████▏ | 44/75 [00:03<00:02, 11.97it/s] 61%|██████████████████████████▎ | 46/75 [00:03<00:02, 11.53it/s] 64%|███████████████████████████▌ | 48/75 [00:03<00:02, 11.00it/s] 67%|████████████████████████████▋ | 50/75 [00:04<00:02, 10.99it/s] 69%|█████████████████████████████▊ | 52/75 [00:04<00:02, 11.42it/s] 72%|██████████████████████████████▉ | 54/75 [00:04<00:01, 10.53it/s] 75%|████████████████████████████████ | 56/75 [00:04<00:01, 11.07it/s] 77%|█████████████████████████████████▎ | 58/75 [00:04<00:01, 11.02it/s] 80%|██████████████████████████████████▍ | 60/75 [00:05<00:01, 11.27it/s] 83%|███████████████████████████████████▌ | 62/75 [00:05<00:01, 11.00it/s] 85%|████████████████████████████████████▋ | 64/75 [00:05<00:00, 11.06it/s] 88%|█████████████████████████████████████▊ | 66/75 [00:05<00:00, 11.62it/s] 91%|██████████████████████████████████████▉ | 68/75 [00:05<00:00, 11.88it/s] 93%|████████████████████████████████████████▏ | 70/75 [00:05<00:00, 12.22it/s] 96%|█████████████████████████████████████████▎ | 72/75 [00:06<00:00, 12.02it/s] {'eval_loss': 1.92901611328125, 'eval_runtime': 6.4241, 'eval_samples_per_second': 11.675, 'eval_steps_per_second': 11.675, 'epoch': 0.0} 0%| | 5/11097 [00:10<2:01:58, 1.52it/s] 100%|███████████████████████████████████████████| 75/75 [00:06<00:00, 12.00it/s] {'loss': 2.3614, 'learning_rate': 0.00012, 'epoch': 0.0} {'loss': 2.0078, 'learning_rate': 0.00014, 'epoch': 0.0} {'loss': 1.4112, 'learning_rate': 0.00016, 'epoch': 0.0} {'loss': 1.7171, 'learning_rate': 0.00018, 'epoch': 0.0} {'loss': 1.4504, 'learning_rate': 0.0002, 'epoch': 0.0} 0%| | 10/11097 [00:13<3:13:53, 1.05s/it] 0%| | 0/75 [00:00<?, ?it/s] 4%|█▊ | 3/75 [00:00<00:03, 22.15it/s] 8%|███▌ | 6/75 [00:00<00:04, 14.71it/s] 11%|████▋ | 8/75 [00:00<00:05, 12.46it/s] 13%|█████▋ | 10/75 [00:00<00:05, 12.58it/s] 16%|██████▉ | 12/75 [00:00<00:04, 12.66it/s] 19%|████████ | 14/75 [00:01<00:04, 12.36it/s] 21%|█████████▏ | 16/75 [00:01<00:04, 12.31it/s] 24%|██████████▎ | 18/75 [00:01<00:04, 12.55it/s] 27%|███████████▍ | 20/75 [00:01<00:04, 12.03it/s] 29%|████████████▌ | 22/75 [00:01<00:04, 11.89it/s] 32%|█████████████▊ | 24/75 [00:01<00:04, 11.44it/s] 35%|██████████████▉ | 26/75 [00:02<00:04, 12.17it/s] 37%|████████████████ | 28/75 [00:02<00:03, 11.99it/s] 40%|█████████████████▏ | 30/75 [00:02<00:03, 11.65it/s] 43%|██████████████████▎ | 32/75 [00:02<00:03, 11.63it/s] 45%|███████████████████▍ | 34/75 [00:02<00:03, 11.55it/s] 48%|████████████████████▋ | 36/75 [00:02<00:03, 11.24it/s] 51%|█████████████████████▊ | 38/75 [00:03<00:03, 11.35it/s] 53%|██████████████████████▉ | 40/75 [00:03<00:03, 11.35it/s] 56%|████████████████████████ | 42/75 [00:03<00:02, 11.37it/s] 59%|█████████████████████████▏ | 44/75 [00:03<00:02, 11.77it/s] 61%|██████████████████████████▎ | 46/75 [00:03<00:02, 11.37it/s] 64%|███████████████████████████▌ | 48/75 [00:04<00:02, 10.84it/s] 67%|████████████████████████████▋ | 50/75 [00:04<00:02, 10.87it/s] 69%|█████████████████████████████▊ | 52/75 [00:04<00:02, 11.30it/s] 72%|██████████████████████████████▉ | 54/75 [00:04<00:02, 10.39it/s] 75%|████████████████████████████████ | 56/75 [00:04<00:01, 10.95it/s] 77%|█████████████████████████████████▎ | 58/75 [00:04<00:01, 10.94it/s] 80%|██████████████████████████████████▍ | 60/75 [00:05<00:01, 11.24it/s] 83%|███████████████████████████████████▌ | 62/75 [00:05<00:01, 10.99it/s] 85%|████████████████████████████████████▋ | 64/75 [00:05<00:00, 11.05it/s] 88%|█████████████████████████████████████▊ | 66/75 [00:05<00:00, 11.65it/s] 91%|██████████████████████████████████████▉ | 68/75 [00:05<00:00, 11.90it/s] 93%|████████████████████████████████████████▏ | 70/75 [00:05<00:00, 12.25it/s] 96%|█████████████████████████████████████████▎ | 72/75 [00:06<00:00, 12.12it/s] {'eval_loss': 1.6972100734710693, 'eval_runtime': 6.4667, 'eval_samples_per_second': 11.598, 'eval_steps_per_second': 11.598, 'epoch': 0.0} 0%| | 10/11097 [00:19<3:13:53, 1.05s/it] 100%|███████████████████████████████████████████| 75/75 [00:06<00:00, 12.06it/s] {'loss': 1.5844, 'learning_rate': 0.00019999999598540582, 'epoch': 0.0} {'loss': 1.3097, 'learning_rate': 0.00019999998394162357, 'epoch': 0.0} {'loss': 2.4036, 'learning_rate': 0.00019999996386865424, 'epoch': 0.0} {'loss': 1.8435, 'learning_rate': 0.0001999999357664994, 'epoch': 0.0} {'loss': 2.3936, 'learning_rate': 0.00019999989963516136, 'epoch': 0.0} 0%| | 15/11097 [00:24<3:52:23, 1.26s/it] 0%| | 0/75 [00:00<?, ?it/s] 4%|█▊ | 3/75 [00:00<00:03, 22.38it/s] 8%|███▌ | 6/75 [00:00<00:04, 14.74it/s] 11%|████▋ | 8/75 [00:00<00:05, 12.46it/s] 13%|█████▋ | 10/75 [00:00<00:05, 12.62it/s] 16%|██████▉ | 12/75 [00:00<00:04, 12.76it/s] 19%|████████ | 14/75 [00:01<00:04, 12.45it/s] 21%|█████████▏ | 16/75 [00:01<00:04, 12.30it/s] 24%|██████████▎ | 18/75 [00:01<00:04, 12.57it/s] 27%|███████████▍ | 20/75 [00:01<00:04, 11.98it/s] 29%|████████████▌ | 22/75 [00:01<00:04, 11.89it/s] 32%|█████████████▊ | 24/75 [00:01<00:04, 11.49it/s] 35%|██████████████▉ | 26/75 [00:02<00:03, 12.26it/s] 37%|████████████████ | 28/75 [00:02<00:03, 12.03it/s] 40%|█████████████████▏ | 30/75 [00:02<00:03, 11.70it/s] 43%|██████████████████▎ | 32/75 [00:02<00:03, 11.70it/s] 45%|███████████████████▍ | 34/75 [00:02<00:03, 11.59it/s] 48%|████████████████████▋ | 36/75 [00:02<00:03, 11.26it/s] 51%|█████████████████████▊ | 38/75 [00:03<00:03, 11.38it/s] 53%|██████████████████████▉ | 40/75 [00:03<00:03, 11.40it/s] 56%|████████████████████████ | 42/75 [00:03<00:02, 11.42it/s] 59%|█████████████████████████▏ | 44/75 [00:03<00:02, 11.80it/s] 61%|██████████████████████████▎ | 46/75 [00:03<00:02, 11.40it/s] 64%|███████████████████████████▌ | 48/75 [00:04<00:02, 10.84it/s] 67%|████████████████████████████▋ | 50/75 [00:04<00:02, 10.83it/s] 69%|█████████████████████████████▊ | 52/75 [00:04<00:02, 11.32it/s] 72%|██████████████████████████████▉ | 54/75 [00:04<00:02, 10.42it/s] 75%|████████████████████████████████ | 56/75 [00:04<00:01, 10.97it/s] 77%|█████████████████████████████████▎ | 58/75 [00:04<00:01, 10.95it/s] 80%|██████████████████████████████████▍ | 60/75 [00:05<00:01, 11.18it/s] 83%|███████████████████████████████████▌ | 62/75 [00:05<00:01, 10.96it/s] 85%|████████████████████████████████████▋ | 64/75 [00:05<00:00, 11.04it/s] 88%|█████████████████████████████████████▊ | 66/75 [00:05<00:00, 11.55it/s] 91%|██████████████████████████████████████▉ | 68/75 [00:05<00:00, 11.81it/s] 93%|████████████████████████████████████████▏ | 70/75 [00:05<00:00, 12.14it/s] 96%|█████████████████████████████████████████▎ | 72/75 [00:06<00:00, 12.03it/s] {'eval_loss': 1.5616062879562378, 'eval_runtime': 6.4677, 'eval_samples_per_second': 11.596, 'eval_steps_per_second': 11.596, 'epoch': 0.0} 0%| | 15/11097 [00:30<3:52:23, 1.26s/it] 100%|███████████████████████████████████████████| 75/75 [00:06<00:00, 11.97it/s] {'loss': 1.1337, 'learning_rate': 0.000199999855474643, 'epoch': 0.0} {'loss': 1.6135, 'learning_rate': 0.00019999980328494788, 'epoch': 0.0} {'loss': 1.7764, 'learning_rate': 0.00019999974306608012, 'epoch': 0.0} {'loss': 1.2131, 'learning_rate': 0.00019999967481804467, 'epoch': 0.01} {'loss': 1.5954, 'learning_rate': 0.00019999959854084692, 'epoch': 0.01} 0%| | 20/11097 [00:33<3:26:42, 1.12s/it] 0%| | 0/75 [00:00<?, ?it/s] 4%|█▊ | 3/75 [00:00<00:03, 22.16it/s] 8%|███▌ | 6/75 [00:00<00:04, 14.70it/s] 11%|████▋ | 8/75 [00:00<00:05, 12.45it/s] 13%|█████▋ | 10/75 [00:00<00:05, 12.62it/s] 16%|██████▉ | 12/75 [00:00<00:04, 12.65it/s] 19%|████████ | 14/75 [00:01<00:04, 12.34it/s] 21%|█████████▏ | 16/75 [00:01<00:04, 12.27it/s] 24%|██████████▎ | 18/75 [00:01<00:04, 12.53it/s] 27%|███████████▍ | 20/75 [00:01<00:04, 11.99it/s] 29%|████████████▌ | 22/75 [00:01<00:04, 11.85it/s] 32%|█████████████▊ | 24/75 [00:01<00:04, 11.44it/s] 35%|██████████████▉ | 26/75 [00:02<00:04, 12.16it/s] 37%|████████████████ | 28/75 [00:02<00:03, 12.00it/s] 40%|█████████████████▏ | 30/75 [00:02<00:03, 11.62it/s] 43%|██████████████████▎ | 32/75 [00:02<00:03, 11.62it/s] 45%|███████████████████▍ | 34/75 [00:02<00:03, 11.52it/s] 48%|████████████████████▋ | 36/75 [00:02<00:03, 11.20it/s] 51%|█████████████████████▊ | 38/75 [00:03<00:03, 11.32it/s] 53%|██████████████████████▉ | 40/75 [00:03<00:03, 11.31it/s] 56%|████████████████████████ | 42/75 [00:03<00:02, 11.35it/s] 59%|█████████████████████████▏ | 44/75 [00:03<00:02, 11.75it/s] 61%|██████████████████████████▎ | 46/75 [00:03<00:02, 11.29it/s] 64%|███████████████████████████▌ | 48/75 [00:04<00:02, 10.79it/s] 67%|████████████████████████████▋ | 50/75 [00:04<00:02, 10.79it/s] 69%|█████████████████████████████▊ | 52/75 [00:04<00:02, 11.24it/s] 72%|██████████████████████████████▉ | 54/75 [00:04<00:02, 10.34it/s] 75%|████████████████████████████████ | 56/75 [00:04<00:01, 10.91it/s] 77%|█████████████████████████████████▎ | 58/75 [00:04<00:01, 10.87it/s] 80%|██████████████████████████████████▍ | 60/75 [00:05<00:01, 11.11it/s] 83%|███████████████████████████████████▌ | 62/75 [00:05<00:01, 10.87it/s] 85%|████████████████████████████████████▋ | 64/75 [00:05<00:01, 10.91it/s] 88%|█████████████████████████████████████▊ | 66/75 [00:05<00:00, 11.49it/s] 91%|██████████████████████████████████████▉ | 68/75 [00:05<00:00, 11.76it/s] 93%|████████████████████████████████████████▏ | 70/75 [00:05<00:00, 12.13it/s] 96%|█████████████████████████████████████████▎ | 72/75 [00:06<00:00, 11.97it/s] {'eval_loss': 1.4697062969207764, 'eval_runtime': 6.5029, 'eval_samples_per_second': 11.533, 'eval_steps_per_second': 11.533, 'epoch': 0.01} 0%| | 20/11097 [00:40<3:26:42, 1.12s/it] 100%|███████████████████████████████████████████| 75/75 [00:06<00:00, 11.91it/s] {'loss': 1.4434, 'learning_rate': 0.00019999951423449303, 'epoch': 0.01} {'loss': 1.276, 'learning_rate': 0.00019999942189898975, 'epoch': 0.01} {'loss': 1.5983, 'learning_rate': 0.00019999932153434453, 'epoch': 0.01} {'loss': 1.2556, 'learning_rate': 0.00019999921314056539, 'epoch': 0.01} {'loss': 1.2912, 'learning_rate': 0.00019999909671766103, 'epoch': 0.01} 0%| | 25/11097 [00:44<3:39:53, 1.19s/it] 0%| | 0/75 [00:00<?, ?it/s] 4%|█▊ | 3/75 [00:00<00:03, 22.06it/s] 8%|███▌ | 6/75 [00:00<00:04, 14.67it/s] 11%|████▋ | 8/75 [00:00<00:05, 12.38it/s] 13%|█████▋ | 10/75 [00:00<00:05, 12.53it/s] 16%|██████▉ | 12/75 [00:00<00:05, 12.59it/s] 19%|████████ | 14/75 [00:01<00:04, 12.31it/s] 21%|█████████▏ | 16/75 [00:01<00:04, 12.23it/s] 24%|██████████▎ | 18/75 [00:01<00:04, 12.45it/s] 27%|███████████▍ | 20/75 [00:01<00:04, 11.88it/s] 29%|████████████▌ | 22/75 [00:01<00:04, 11.73it/s] 32%|█████████████▊ | 24/75 [00:01<00:04, 11.35it/s] 35%|██████████████▉ | 26/75 [00:02<00:04, 12.08it/s] 37%|████████████████ | 28/75 [00:02<00:03, 11.89it/s] 40%|█████████████████▏ | 30/75 [00:02<00:03, 11.52it/s] 43%|██████████████████▎ | 32/75 [00:02<00:03, 11.53it/s] 45%|███████████████████▍ | 34/75 [00:02<00:03, 11.41it/s] 48%|████████████████████▋ | 36/75 [00:02<00:03, 11.11it/s] 51%|█████████████████████▊ | 38/75 [00:03<00:03, 11.24it/s] 53%|██████████████████████▉ | 40/75 [00:03<00:03, 11.27it/s] 56%|████████████████████████ | 42/75 [00:03<00:02, 11.26it/s] 59%|█████████████████████████▏ | 44/75 [00:03<00:02, 11.66it/s] 61%|██████████████████████████▎ | 46/75 [00:03<00:02, 11.23it/s] 64%|███████████████████████████▌ | 48/75 [00:04<00:02, 10.72it/s] 67%|████████████████████████████▋ | 50/75 [00:04<00:02, 10.76it/s] 69%|█████████████████████████████▊ | 52/75 [00:04<00:02, 11.17it/s] 72%|██████████████████████████████▉ | 54/75 [00:04<00:02, 10.27it/s] 75%|████████████████████████████████ | 56/75 [00:04<00:01, 10.81it/s] 77%|█████████████████████████████████▎ | 58/75 [00:05<00:01, 10.81it/s] 80%|██████████████████████████████████▍ | 60/75 [00:05<00:01, 11.09it/s] 83%|███████████████████████████████████▌ | 62/75 [00:05<00:01, 10.81it/s] 85%|████████████████████████████████████▋ | 64/75 [00:05<00:01, 10.86it/s] 88%|█████████████████████████████████████▊ | 66/75 [00:05<00:00, 11.44it/s] 91%|██████████████████████████████████████▉ | 68/75 [00:05<00:00, 11.68it/s] 93%|████████████████████████████████████████▏ | 70/75 [00:06<00:00, 11.99it/s] 96%|█████████████████████████████████████████▎ | 72/75 [00:06<00:00, 11.85it/s] {'eval_loss': 1.4360172748565674, 'eval_runtime': 6.5483, 'eval_samples_per_second': 11.453, 'eval_steps_per_second': 11.453, 'epoch': 0.01} 0%| | 25/11097 [00:50<3:39:53, 1.19s/it] 100%|███████████████████████████████████████████| 75/75 [00:06<00:00, 11.80it/s] {'loss': 1.4833, 'learning_rate': 0.00019999897226564083, 'epoch': 0.01} {'loss': 1.5938, 'learning_rate': 0.0001999988397845148, 'epoch': 0.01} {'loss': 1.4827, 'learning_rate': 0.00019999869927429353, 'epoch': 0.01} {'loss': 1.195, 'learning_rate': 0.0001999985507349883, 'epoch': 0.01} {'loss': 1.2247, 'learning_rate': 0.00019999839416661106, 'epoch': 0.01} 0%| | 30/11097 [00:53<3:36:20, 1.17s/it] 0%| | 0/75 [00:00<?, ?it/s] 4%|█▊ | 3/75 [00:00<00:03, 22.00it/s] 8%|███▌ | 6/75 [00:00<00:04, 14.47it/s] 11%|████▋ | 8/75 [00:00<00:05, 12.31it/s] 13%|█████▋ | 10/75 [00:00<00:05, 12.44it/s] 16%|██████▉ | 12/75 [00:00<00:04, 12.60it/s] 19%|████████ | 14/75 [00:01<00:04, 12.31it/s] 21%|█████████▏ | 16/75 [00:01<00:04, 12.18it/s] 24%|██████████▎ | 18/75 [00:01<00:04, 12.43it/s] 27%|███████████▍ | 20/75 [00:01<00:04, 11.90it/s] 29%|████████████▌ | 22/75 [00:01<00:04, 11.82it/s] 32%|█████████████▊ | 24/75 [00:01<00:04, 11.44it/s] 35%|██████████████▉ | 26/75 [00:02<00:04, 12.17it/s] 37%|████████████████ | 28/75 [00:02<00:03, 11.92it/s] 40%|█████████████████▏ | 30/75 [00:02<00:03, 11.60it/s] 43%|██████████████████▎ | 32/75 [00:02<00:03, 11.57it/s] 45%|███████████████████▍ | 34/75 [00:02<00:03, 11.52it/s] 48%|████████████████████▋ | 36/75 [00:02<00:03, 11.19it/s] 51%|█████████████████████▊ | 38/75 [00:03<00:03, 11.32it/s] 53%|██████████████████████▉ | 40/75 [00:03<00:03, 11.31it/s] 56%|████████████████████████ | 42/75 [00:03<00:02, 11.31it/s] 59%|█████████████████████████▏ | 44/75 [00:03<00:02, 11.67it/s] 61%|██████████████████████████▎ | 46/75 [00:03<00:02, 11.26it/s] 64%|███████████████████████████▌ | 48/75 [00:04<00:02, 10.75it/s] 67%|████████████████████████████▋ | 50/75 [00:04<00:02, 10.76it/s] 69%|█████████████████████████████▊ | 52/75 [00:04<00:02, 11.18it/s] 72%|██████████████████████████████▉ | 54/75 [00:04<00:02, 10.29it/s] 75%|████████████████████████████████ | 56/75 [00:04<00:01, 10.84it/s] 77%|█████████████████████████████████▎ | 58/75 [00:04<00:01, 10.86it/s] 80%|██████████████████████████████████▍ | 60/75 [00:05<00:01, 11.14it/s] 83%|███████████████████████████████████▌ | 62/75 [00:05<00:01, 10.87it/s] 85%|████████████████████████████████████▋ | 64/75 [00:05<00:01, 10.90it/s] 88%|█████████████████████████████████████▊ | 66/75 [00:05<00:00, 11.45it/s] 91%|██████████████████████████████████████▉ | 68/75 [00:05<00:00, 11.70it/s] 93%|████████████████████████████████████████▏ | 70/75 [00:06<00:00, 12.04it/s] 96%|█████████████████████████████████████████▎ | 72/75 [00:06<00:00, 11.88it/s] {'eval_loss': 1.3883757591247559, 'eval_runtime': 6.532, 'eval_samples_per_second': 11.482, 'eval_steps_per_second': 11.482, 'epoch': 0.01} 0%| | 30/11097 [01:00<3:36:20, 1.17s/it] 100%|███████████████████████████████████████████| 75/75 [00:06<00:00, 11.84it/s] {'loss': 1.0079, 'learning_rate': 0.00019999822956917439, 'epoch': 0.01} {'loss': 1.6677, 'learning_rate': 0.00019999805694269145, 'epoch': 0.01} {'loss': 1.4063, 'learning_rate': 0.0001999978762871762, 'epoch': 0.01} {'loss': 1.9272, 'learning_rate': 0.00019999768760264303, 'epoch': 0.01} {'loss': 1.1411, 'learning_rate': 0.00019999749088910718, 'epoch': 0.01} 0%| | 35/11097 [01:04<3:51:36, 1.26s/it] 0%| | 0/75 [00:00<?, ?it/s] 4%|█▊ | 3/75 [00:00<00:03, 22.06it/s] 8%|███▌ | 6/75 [00:00<00:04, 14.53it/s] 11%|████▋ | 8/75 [00:00<00:05, 12.32it/s] 13%|█████▋ | 10/75 [00:00<00:05, 12.43it/s] 16%|██████▉ | 12/75 [00:00<00:05, 12.45it/s] 19%|████████ | 14/75 [00:01<00:04, 12.20it/s] 21%|█████████▏ | 16/75 [00:01<00:04, 12.12it/s] 24%|██████████▎ | 18/75 [00:01<00:04, 12.32it/s] 27%|███████████▍ | 20/75 [00:01<00:04, 11.80it/s] 29%|████████████▌ | 22/75 [00:01<00:04, 11.65it/s] 32%|█████████████▊ | 24/75 [00:01<00:04, 11.24it/s] 35%|██████████████▉ | 26/75 [00:02<00:04, 12.00it/s] 37%|████████████████ | 28/75 [00:02<00:03, 11.85it/s] 40%|█████████████████▏ | 30/75 [00:02<00:03, 11.49it/s] 43%|██████████████████▎ | 32/75 [00:02<00:03, 11.49it/s] 45%|███████████████████▍ | 34/75 [00:02<00:03, 11.36it/s] 48%|████████████████████▋ | 36/75 [00:03<00:03, 11.03it/s] 51%|█████████████████████▊ | 38/75 [00:03<00:03, 11.20it/s] 53%|██████████████████████▉ | 40/75 [00:03<00:03, 11.19it/s] 56%|████████████████████████ | 42/75 [00:03<00:02, 11.22it/s] 59%|█████████████████████████▏ | 44/75 [00:03<00:02, 11.61it/s] 61%|██████████████████████████▎ | 46/75 [00:03<00:02, 11.15it/s] 64%|███████████████████████████▌ | 48/75 [00:04<00:02, 10.66it/s] 67%|████████████████████████████▋ | 50/75 [00:04<00:02, 10.66it/s] 69%|█████████████████████████████▊ | 52/75 [00:04<00:02, 11.11it/s] 72%|██████████████████████████████▉ | 54/75 [00:04<00:02, 10.21it/s] 75%|████████████████████████████████ | 56/75 [00:04<00:01, 10.76it/s] 77%|█████████████████████████████████▎ | 58/75 [00:05<00:01, 10.75it/s] 80%|██████████████████████████████████▍ | 60/75 [00:05<00:01, 11.04it/s] 83%|███████████████████████████████████▌ | 62/75 [00:05<00:01, 10.81it/s] 85%|████████████████████████████████████▋ | 64/75 [00:05<00:01, 10.83it/s] 88%|█████████████████████████████████████▊ | 66/75 [00:05<00:00, 11.37it/s] 91%|██████████████████████████████████████▉ | 68/75 [00:05<00:00, 11.60it/s] 93%|████████████████████████████████████████▏ | 70/75 [00:06<00:00, 11.96it/s] 96%|█████████████████████████████████████████▎ | 72/75 [00:06<00:00, 11.78it/s] {'eval_loss': 1.372732162475586, 'eval_runtime': 6.5827, 'eval_samples_per_second': 11.393, 'eval_steps_per_second': 11.393, 'epoch': 0.01} 0%| | 35/11097 [01:11<3:51:36, 1.26s/it] 100%|███████████████████████████████████████████| 75/75 [00:06<00:00, 11.77it/s] {'loss': 1.5162, 'learning_rate': 0.0001999972861465844, 'epoch': 0.01} {'loss': 1.8269, 'learning_rate': 0.00019999707337509115, 'epoch': 0.01} {'loss': 1.079, 'learning_rate': 0.00019999685257464446, 'epoch': 0.01} {'loss': 1.5693, 'learning_rate': 0.00019999662374526215, 'epoch': 0.01} {'loss': 1.2695, 'learning_rate': 0.00019999638688696254, 'epoch': 0.01} 0%|▏ | 40/11097 [01:14<3:32:50, 1.16s/it] 0%| | 0/75 [00:00<?, ?it/s] 4%|█▊ | 3/75 [00:00<00:03, 21.90it/s] 8%|███▌ | 6/75 [00:00<00:04, 14.27it/s] 11%|████▋ | 8/75 [00:00<00:05, 12.14it/s] 13%|█████▋ | 10/75 [00:00<00:05, 12.28it/s] 16%|██████▉ | 12/75 [00:00<00:05, 12.40it/s] 19%|████████ | 14/75 [00:01<00:05, 12.15it/s] 21%|█████████▏ | 16/75 [00:01<00:04, 12.00it/s] 24%|██████████▎ | 18/75 [00:01<00:04, 12.21it/s] 27%|███████████▍ | 20/75 [00:01<00:04, 11.71it/s] 29%|████████████▌ | 22/75 [00:01<00:04, 11.54it/s] 32%|█████████████▊ | 24/75 [00:01<00:04, 11.21it/s] 35%|██████████████▉ | 26/75 [00:02<00:04, 11.94it/s] 37%|████████████████ | 28/75 [00:02<00:03, 11.76it/s] 40%|█████████████████▏ | 30/75 [00:02<00:03, 11.40it/s] 43%|██████████████████▎ | 32/75 [00:02<00:03, 11.35it/s] 45%|███████████████████▍ | 34/75 [00:02<00:03, 11.36it/s] 48%|████████████████████▋ | 36/75 [00:03<00:03, 11.06it/s] 51%|█████████████████████▊ | 38/75 [00:03<00:03, 11.18it/s] 53%|██████████████████████▉ | 40/75 [00:03<00:03, 11.17it/s] 56%|████████████████████████ | 42/75 [00:03<00:02, 11.10it/s] 59%|█████████████████████████▏ | 44/75 [00:03<00:02, 11.53it/s] 61%|██████████████████████████▎ | 46/75 [00:03<00:02, 11.10it/s] 64%|███████████████████████████▌ | 48/75 [00:04<00:02, 10.61it/s] 67%|████████████████████████████▋ | 50/75 [00:04<00:02, 10.57it/s] 69%|█████████████████████████████▊ | 52/75 [00:04<00:02, 10.98it/s] 72%|██████████████████████████████▉ | 54/75 [00:04<00:02, 10.09it/s] 75%|████████████████████████████████ | 56/75 [00:04<00:01, 10.65it/s] 77%|█████████████████████████████████▎ | 58/75 [00:05<00:01, 10.70it/s] 80%|██████████████████████████████████▍ | 60/75 [00:05<00:01, 10.93it/s] 83%|███████████████████████████████████▌ | 62/75 [00:05<00:01, 10.67it/s] 85%|████████████████████████████████████▋ | 64/75 [00:05<00:01, 10.72it/s] 88%|█████████████████████████████████████▊ | 66/75 [00:05<00:00, 11.31it/s] 91%|██████████████████████████████████████▉ | 68/75 [00:05<00:00, 11.55it/s] 93%|████████████████████████████████████████▏ | 70/75 [00:06<00:00, 11.90it/s] 96%|█████████████████████████████████████████▎ | 72/75 [00:06<00:00, 11.79it/s] {'eval_loss': 1.3437153100967407, 'eval_runtime': 6.6275, 'eval_samples_per_second': 11.316, 'eval_steps_per_second': 11.316, 'epoch': 0.01} 0%|▏ | 40/11097 [01:20<3:32:50, 1.16s/it] 100%|███████████████████████████████████████████| 75/75 [00:06<00:00, 11.73it/s] {'loss': 1.1865, 'learning_rate': 0.00019999614199976465, 'epoch': 0.01} {'loss': 1.5142, 'learning_rate': 0.00019999588908368817, 'epoch': 0.01} {'loss': 0.9165, 'learning_rate': 0.00019999562813875334, 'epoch': 0.01} {'loss': 1.0933, 'learning_rate': 0.00019999535916498116, 'epoch': 0.01} {'loss': 0.9505, 'learning_rate': 0.00019999508216239322, 'epoch': 0.01} 0%|▏ | 45/11097 [01:25<3:56:41, 1.29s/it] 0%| | 0/75 [00:00<?, ?it/s] 4%|█▊ | 3/75 [00:00<00:03, 21.63it/s] 8%|███▌ | 6/75 [00:00<00:04, 14.24it/s] 11%|████▋ | 8/75 [00:00<00:05, 12.12it/s] 13%|█████▋ | 10/75 [00:00<00:05, 12.28it/s] 16%|██████▉ | 12/75 [00:00<00:05, 12.31it/s] 19%|████████ | 14/75 [00:01<00:05, 12.14it/s] 21%|█████████▏ | 16/75 [00:01<00:04, 11.97it/s] 24%|██████████▎ | 18/75 [00:01<00:04, 12.23it/s] 27%|███████████▍ | 20/75 [00:01<00:04, 11.74it/s] 29%|████████████▌ | 22/75 [00:01<00:04, 11.52it/s] 32%|█████████████▊ | 24/75 [00:01<00:04, 11.20it/s] 35%|██████████████▉ | 26/75 [00:02<00:04, 11.91it/s] 37%|████████████████ | 28/75 [00:02<00:04, 11.70it/s] 40%|█████████████████▏ | 30/75 [00:02<00:03, 11.36it/s] 43%|██████████████████▎ | 32/75 [00:02<00:03, 11.34it/s] 45%|███████████████████▍ | 34/75 [00:02<00:03, 11.22it/s] 48%|████████████████████▋ | 36/75 [00:03<00:03, 10.98it/s] 51%|█████████████████████▊ | 38/75 [00:03<00:03, 11.09it/s] 53%|██████████████████████▉ | 40/75 [00:03<00:03, 11.09it/s] 56%|████████████████████████ | 42/75 [00:03<00:02, 11.11it/s] 59%|█████████████████████████▏ | 44/75 [00:03<00:02, 11.50it/s] 61%|██████████████████████████▎ | 46/75 [00:03<00:02, 11.07it/s] 64%|███████████████████████████▌ | 48/75 [00:04<00:02, 10.56it/s] 67%|████████████████████████████▋ | 50/75 [00:04<00:02, 10.58it/s] 69%|█████████████████████████████▊ | 52/75 [00:04<00:02, 10.98it/s] 72%|██████████████████████████████▉ | 54/75 [00:04<00:02, 10.09it/s] 75%|████████████████████████████████ | 56/75 [00:04<00:01, 10.63it/s] 77%|█████████████████████████████████▎ | 58/75 [00:05<00:01, 10.61it/s] 80%|██████████████████████████████████▍ | 60/75 [00:05<00:01, 10.89it/s] 83%|███████████████████████████████████▌ | 62/75 [00:05<00:01, 10.67it/s] 85%|████████████████████████████████████▋ | 64/75 [00:05<00:01, 10.72it/s] 88%|█████████████████████████████████████▊ | 66/75 [00:05<00:00, 11.28it/s] 91%|██████████████████████████████████████▉ | 68/75 [00:05<00:00, 11.49it/s] 93%|████████████████████████████████████████▏ | 70/75 [00:06<00:00, 11.83it/s] 96%|█████████████████████████████████████████▎ | 72/75 [00:06<00:00, 11.68it/s] {'eval_loss': 1.328573226928711, 'eval_runtime': 6.6502, 'eval_samples_per_second': 11.278, 'eval_steps_per_second': 11.278, 'epoch': 0.01} 0%|▏ | 45/11097 [01:32<3:56:41, 1.29s/it] 100%|███████████████████████████████████████████| 75/75 [00:06<00:00, 11.66it/s] {'loss': 1.103, 'learning_rate': 0.0001999947971310118, 'epoch': 0.01} {'loss': 1.3633, 'learning_rate': 0.0001999945040708597, 'epoch': 0.01} {'loss': 1.6129, 'learning_rate': 0.00019999420298196055, 'epoch': 0.01} {'loss': 1.291, 'learning_rate': 0.00019999389386433843, 'epoch': 0.01} {'loss': 0.9853, 'learning_rate': 0.00019999357671801818, 'epoch': 0.01} 0%|▏ | 50/11097 [01:35<3:39:31, 1.19s/it] 0%| | 0/75 [00:00<?, ?it/s] 4%|█▊ | 3/75 [00:00<00:03, 21.71it/s] 8%|███▌ | 6/75 [00:00<00:04, 14.32it/s] 11%|████▋ | 8/75 [00:00<00:05, 12.10it/s] 13%|█████▋ | 10/75 [00:00<00:05, 12.25it/s] 16%|██████▉ | 12/75 [00:00<00:05, 12.27it/s] 19%|████████ | 14/75 [00:01<00:05, 12.02it/s] 21%|█████████▏ | 16/75 [00:01<00:04, 11.93it/s] 24%|██████████▎ | 18/75 [00:01<00:04, 12.16it/s] 27%|███████████▍ | 20/75 [00:01<00:04, 11.60it/s] 29%|████████████▌ | 22/75 [00:01<00:04, 11.43it/s] 32%|█████████████▊ | 24/75 [00:01<00:04, 11.08it/s] 35%|██████████████▉ | 26/75 [00:02<00:04, 11.84it/s] 37%|████████████████ | 28/75 [00:02<00:04, 11.71it/s] 40%|█████████████████▏ | 30/75 [00:02<00:03, 11.28it/s] 43%|██████████████████▎ | 32/75 [00:02<00:03, 11.30it/s] 45%|███████████████████▍ | 34/75 [00:02<00:03, 11.15it/s] 48%|████████████████████▋ | 36/75 [00:03<00:03, 10.92it/s] 51%|█████████████████████▊ | 38/75 [00:03<00:03, 11.07it/s] 53%|██████████████████████▉ | 40/75 [00:03<00:03, 11.08it/s] 56%|████████████████████████ | 42/75 [00:03<00:02, 11.03it/s] 59%|█████████████████████████▏ | 44/75 [00:03<00:02, 11.39it/s] 61%|██████████████████████████▎ | 46/75 [00:03<00:02, 10.95it/s] 64%|███████████████████████████▌ | 48/75 [00:04<00:02, 10.49it/s] 67%|████████████████████████████▋ | 50/75 [00:04<00:02, 10.49it/s] 69%|█████████████████████████████▊ | 52/75 [00:04<00:02, 10.89it/s] 72%|██████████████████████████████▉ | 54/75 [00:04<00:02, 10.05it/s] 75%|████████████████████████████████ | 56/75 [00:04<00:01, 10.57it/s] 77%|█████████████████████████████████▎ | 58/75 [00:05<00:01, 10.55it/s] 80%|██████████████████████████████████▍ | 60/75 [00:05<00:01, 10.84it/s] 83%|███████████████████████████████████▌ | 62/75 [00:05<00:01, 10.57it/s] 85%|████████████████████████████████████▋ | 64/75 [00:05<00:01, 10.62it/s] 88%|█████████████████████████████████████▊ | 66/75 [00:05<00:00, 11.17it/s] 91%|██████████████████████████████████████▉ | 68/75 [00:05<00:00, 11.44it/s] 93%|████████████████████████████████████████▏ | 70/75 [00:06<00:00, 11.79it/s] 96%|█████████████████████████████████████████▎ | 72/75 [00:06<00:00, 11.62it/s] {'eval_loss': 1.3150309324264526, 'eval_runtime': 6.6927, 'eval_samples_per_second': 11.206, 'eval_steps_per_second': 11.206, 'epoch': 0.01} 0%|▏ | 50/11097 [01:42<3:39:31, 1.19s/it] 100%|███████████████████████████████████████████| 75/75 [00:06<00:00, 11.57it/s] {'loss': 1.2576, 'learning_rate': 0.00019999325154302535, 'epoch': 0.01} {'loss': 1.3121, 'learning_rate': 0.00019999291833938597, 'epoch': 0.01} {'loss': 1.105, 'learning_rate': 0.0001999925771071268, 'epoch': 0.01} {'loss': 1.1401, 'learning_rate': 0.00019999222784627523, 'epoch': 0.01} {'loss': 1.2429, 'learning_rate': 0.00019999187055685935, 'epoch': 0.01} 0%|▏ | 55/11097 [01:46<4:03:31, 1.32s/it] 0%| | 0/75 [00:00<?, ?it/s] 4%|█▊ | 3/75 [00:00<00:03, 21.64it/s] 8%|███▌ | 6/75 [00:00<00:04, 14.03it/s] 11%|████▋ | 8/75 [00:00<00:05, 11.91it/s] 13%|█████▋ | 10/75 [00:00<00:05, 12.17it/s] 16%|██████▉ | 12/75 [00:00<00:05, 12.24it/s] 19%|████████ | 14/75 [00:01<00:05, 12.01it/s] 21%|█████████▏ | 16/75 [00:01<00:04, 11.83it/s] 24%|██████████▎ | 18/75 [00:01<00:04, 12.07it/s] 27%|███████████▍ | 20/75 [00:01<00:04, 11.53it/s] 29%|████████████▌ | 22/75 [00:01<00:04, 11.41it/s] 32%|█████████████▊ | 24/75 [00:02<00:04, 11.11it/s] 35%|██████████████▉ | 26/75 [00:02<00:04, 11.77it/s] 37%|████████████████ | 28/75 [00:02<00:04, 11.60it/s] 40%|█████████████████▏ | 30/75 [00:02<00:04, 11.21it/s] 43%|██████████████████▎ | 32/75 [00:02<00:03, 11.22it/s] 45%|███████████████████▍ | 34/75 [00:02<00:03, 11.18it/s] 48%|████████████████████▋ | 36/75 [00:03<00:03, 10.93it/s] 51%|█████████████████████▊ | 38/75 [00:03<00:03, 11.04it/s] 53%|██████████████████████▉ | 40/75 [00:03<00:03, 11.01it/s] 56%|████████████████████████ | 42/75 [00:03<00:03, 10.99it/s] 59%|█████████████████████████▏ | 44/75 [00:03<00:02, 11.35it/s] 61%|██████████████████████████▎ | 46/75 [00:03<00:02, 10.95it/s] 64%|███████████████████████████▌ | 48/75 [00:04<00:02, 10.48it/s] 67%|████████████████████████████▋ | 50/75 [00:04<00:02, 10.47it/s] 69%|█████████████████████████████▊ | 52/75 [00:04<00:02, 10.85it/s] 72%|██████████████████████████████▉ | 54/75 [00:04<00:02, 9.99it/s] 75%|████████████████████████████████ | 56/75 [00:04<00:01, 10.54it/s] 77%|█████████████████████████████████▎ | 58/75 [00:05<00:01, 10.54it/s] 80%|██████████████████████████████████▍ | 60/75 [00:05<00:01, 10.81it/s] 83%|███████████████████████████████████▌ | 62/75 [00:05<00:01, 10.52it/s] 85%|████████████████████████████████████▋ | 64/75 [00:05<00:01, 10.55it/s] 88%|█████████████████████████████████████▊ | 66/75 [00:05<00:00, 11.14it/s] 91%|██████████████████████████████████████▉ | 68/75 [00:06<00:00, 11.39it/s] 93%|████████████████████████████████████████▏ | 70/75 [00:06<00:00, 11.72it/s] 96%|█████████████████████████████████████████▎ | 72/75 [00:06<00:00, 11.59it/s] {'eval_loss': 1.295883297920227, 'eval_runtime': 6.7224, 'eval_samples_per_second': 11.157, 'eval_steps_per_second': 11.157, 'epoch': 0.01} 0%|▏ | 55/11097 [01:53<4:03:31, 1.32s/it] 100%|███████████████████████████████████████████| 75/75 [00:06<00:00, 11.49it/s] {'loss': 0.9401, 'learning_rate': 0.0001999915052389078, 'epoch': 0.02} {'loss': 1.1006, 'learning_rate': 0.00019999113189244993, 'epoch': 0.02} {'loss': 1.0839, 'learning_rate': 0.00019999075051751572, 'epoch': 0.02} {'loss': 1.5278, 'learning_rate': 0.0001999903611141358, 'epoch': 0.02} {'loss': 1.161, 'learning_rate': 0.00019998996368234142, 'epoch': 0.02} 1%|▏ | 60/11097 [01:56<3:33:59, 1.16s/it] 0%| | 0/75 [00:00<?, ?it/s] 4%|█▊ | 3/75 [00:00<00:03, 21.40it/s] 8%|███▌ | 6/75 [00:00<00:04, 14.00it/s] 11%|████▋ | 8/75 [00:00<00:05, 11.91it/s] 13%|█████▋ | 10/75 [00:00<00:05, 12.08it/s] 16%|██████▉ | 12/75 [00:00<00:05, 12.20it/s] 19%|████████ | 14/75 [00:01<00:05, 11.91it/s] 21%|█████████▏ | 16/75 [00:01<00:04, 11.81it/s] 24%|██████████▎ | 18/75 [00:01<00:04, 12.01it/s] 27%|███████████▍ | 20/75 [00:01<00:04, 11.48it/s] 29%|████████████▌ | 22/75 [00:01<00:04, 11.35it/s] 32%|█████████████▊ | 24/75 [00:02<00:04, 11.00it/s] 35%|██████████████▉ | 26/75 [00:02<00:04, 11.74it/s] 37%|████████████████ | 28/75 [00:02<00:04, 11.51it/s] 40%|█████████████████▏ | 30/75 [00:02<00:04, 11.10it/s] 43%|██████████████████▎ | 32/75 [00:02<00:03, 11.16it/s] 45%|███████████████████▍ | 34/75 [00:02<00:03, 11.06it/s] 48%|████████████████████▋ | 36/75 [00:03<00:03, 10.83it/s] 51%|█████████████████████▊ | 38/75 [00:03<00:03, 10.94it/s] 53%|██████████████████████▉ | 40/75 [00:03<00:03, 10.92it/s] 56%|████████████████████████ | 42/75 [00:03<00:03, 10.95it/s] 59%|█████████████████████████▏ | 44/75 [00:03<00:02, 11.33it/s] 61%|██████████████████████████▎ | 46/75 [00:03<00:02, 10.88it/s] 64%|███████████████████████████▌ | 48/75 [00:04<00:02, 10.39it/s] 67%|████████████████████████████▋ | 50/75 [00:04<00:02, 10.37it/s] 69%|█████████████████████████████▊ | 52/75 [00:04<00:02, 10.79it/s] 72%|██████████████████████████████▉ | 54/75 [00:04<00:02, 9.97it/s] 75%|████████████████████████████████ | 56/75 [00:04<00:01, 10.49it/s] 77%|█████████████████████████████████▎ | 58/75 [00:05<00:01, 10.49it/s] 80%|██████████████████████████████████▍ | 60/75 [00:05<00:01, 10.72it/s] 83%|███████████████████████████████████▌ | 62/75 [00:05<00:01, 10.47it/s] 85%|████████████████████████████████████▋ | 64/75 [00:05<00:01, 10.54it/s] 88%|█████████████████████████████████████▊ | 66/75 [00:05<00:00, 11.08it/s] 91%|██████████████████████████████████████▉ | 68/75 [00:06<00:00, 11.31it/s] 93%|████████████████████████████████████████▏ | 70/75 [00:06<00:00, 11.63it/s] 96%|█████████████████████████████████████████▎ | 72/75 [00:06<00:00, 11.52it/s] {'eval_loss': 1.280275583267212, 'eval_runtime': 6.7603, 'eval_samples_per_second': 11.094, 'eval_steps_per_second': 11.094, 'epoch': 0.02} 1%|▏ | 60/11097 [02:03<3:33:59, 1.16s/it] 100%|███████████████████████████████████████████| 75/75 [00:06<00:00, 11.45it/s] {'loss': 1.6926, 'learning_rate': 0.00019998955822216448, 'epoch': 0.02} {'loss': 1.0253, 'learning_rate': 0.00019998914473363754, 'epoch': 0.02} {'loss': 1.2151, 'learning_rate': 0.00019998872321679386, 'epoch': 0.02} {'loss': 1.1427, 'learning_rate': 0.00019998829367166718, 'epoch': 0.02} {'loss': 1.3025, 'learning_rate': 0.00019998785609829205, 'epoch': 0.02} 1%|▏ | 65/11097 [02:07<3:49:55, 1.25s/it] 0%| | 0/75 [00:00<?, ?it/s] 4%|█▊ | 3/75 [00:00<00:03, 21.53it/s] 8%|███▌ | 6/75 [00:00<00:04, 14.07it/s] 11%|████▋ | 8/75 [00:00<00:05, 11.93it/s] 13%|█████▋ | 10/75 [00:00<00:05, 12.10it/s] 16%|██████▉ | 12/75 [00:00<00:05, 12.13it/s] 19%|████████ | 14/75 [00:01<00:05, 11.89it/s] 21%|█████████▏ | 16/75 [00:01<00:04, 11.81it/s] 24%|██████████▎ | 18/75 [00:01<00:04, 12.01it/s] 27%|███████████▍ | 20/75 [00:01<00:04, 11.48it/s] 29%|████████████▌ | 22/75 [00:01<00:04, 11.33it/s] 32%|█████████████▊ | 24/75 [00:02<00:04, 11.01it/s] 35%|██████████████▉ | 26/75 [00:02<00:04, 11.72it/s] 37%|████████████████ | 28/75 [00:02<00:04, 11.50it/s] 40%|█████████████████▏ | 30/75 [00:02<00:04, 11.11it/s] 43%|██████████████████▎ | 32/75 [00:02<00:03, 11.16it/s] 45%|███████████████████▍ | 34/75 [00:02<00:03, 11.05it/s] 48%|████████████████████▋ | 36/75 [00:03<00:03, 10.77it/s] 51%|█████████████████████▊ | 38/75 [00:03<00:03, 10.93it/s] 53%|██████████████████████▉ | 40/75 [00:03<00:03, 10.91it/s] 56%|████████████████████████ | 42/75 [00:03<00:03, 10.90it/s] 59%|█████████████████████████▏ | 44/75 [00:03<00:02, 11.29it/s] 61%|██████████████████████████▎ | 46/75 [00:04<00:02, 10.81it/s] 64%|███████████████████████████▌ | 48/75 [00:04<00:02, 10.38it/s] 67%|████████████████████████████▋ | 50/75 [00:04<00:02, 10.37it/s] 69%|█████████████████████████████▊ | 52/75 [00:04<00:02, 10.75it/s] 72%|██████████████████████████████▉ | 54/75 [00:04<00:02, 9.92it/s] 75%|████████████████████████████████ | 56/75 [00:04<00:01, 10.44it/s] 77%|█████████████████████████████████▎ | 58/75 [00:05<00:01, 10.47it/s] 80%|██████████████████████████████████▍ | 60/75 [00:05<00:01, 10.71it/s] 83%|███████████████████████████████████▌ | 62/75 [00:05<00:01, 10.46it/s] 85%|████████████████████████████████████▋ | 64/75 [00:05<00:01, 10.53it/s] 88%|█████████████████████████████████████▊ | 66/75 [00:05<00:00, 11.05it/s] 91%|██████████████████████████████████████▉ | 68/75 [00:06<00:00, 11.28it/s] 93%|████████████████████████████████████████▏ | 70/75 [00:06<00:00, 11.61it/s] 96%|█████████████████████████████████████████▎ | 72/75 [00:06<00:00, 11.51it/s] 99%|██████████████████████████████████████████▍| 74/75 [00:06<00:00, 11.40it/s]^C 1%|▏ | 65/11097 [02:14<6:21:22, 2.07s/it]
In [ ]: