In [1]:
!git clone https://github.com/OpenAccess-AI-Collective/axolotl.git
Cloning into 'axolotl'... remote: Enumerating objects: 2614, done. remote: Counting objects: 100% (890/890), done. remote: Compressing objects: 100% (195/195), done. remote: Total 2614 (delta 791), reused 721 (delta 685), pack-reused 1724 Receiving objects: 100% (2614/2614), 1.33 MiB | 9.54 MiB/s, done. Resolving deltas: 100% (1627/1627), done.
In [2]:
%cd axolotl
/content/axolotl
In [3]:
!pip install -e .
Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/
Obtaining file:///content/axolotl
Preparing metadata (setup.py) ... done
Collecting transformers@ git+https://github.com/huggingface/transformers.git (from axolotl==0.1)
Cloning https://github.com/huggingface/transformers.git to /tmp/pip-install-esok7o5q/transformers_6129b103593a488c99b68220b5a40d08
Running command git clone --filter=blob:none --quiet https://github.com/huggingface/transformers.git /tmp/pip-install-esok7o5q/transformers_6129b103593a488c99b68220b5a40d08
Resolved https://github.com/huggingface/transformers.git to commit 8f093fb799246f7dd9104ff44728da0c53a9f67a
Installing build dependencies ... done
Getting requirements to build wheel ... done
Preparing metadata (pyproject.toml) ... done
Collecting bitsandbytes>=0.39.0 (from axolotl==0.1)
Downloading bitsandbytes-0.39.0-py3-none-any.whl (92.2 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 92.2/92.2 MB 10.6 MB/s eta 0:00:00
Collecting accelerate (from axolotl==0.1)
Downloading accelerate-0.20.3-py3-none-any.whl (227 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 227.6/227.6 kB 26.1 MB/s eta 0:00:00
Collecting addict (from axolotl==0.1)
Downloading addict-2.4.0-py3-none-any.whl (3.8 kB)
Collecting fire (from axolotl==0.1)
Downloading fire-0.5.0.tar.gz (88 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 88.3/88.3 kB 12.1 MB/s eta 0:00:00
Preparing metadata (setup.py) ... done
Requirement already satisfied: PyYAML==6.0 in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (6.0)
Collecting datasets (from axolotl==0.1)
Downloading datasets-2.12.0-py3-none-any.whl (474 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 474.6/474.6 kB 52.0 MB/s eta 0:00:00
Collecting sentencepiece (from axolotl==0.1)
Downloading sentencepiece-0.1.99-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.3/1.3 MB 80.8 MB/s eta 0:00:00
Collecting wandb (from axolotl==0.1)
Downloading wandb-0.15.4-py3-none-any.whl (2.1 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 2.1/2.1 MB 101.9 MB/s eta 0:00:00
Collecting einops (from axolotl==0.1)
Downloading einops-0.6.1-py3-none-any.whl (42 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 42.2/42.2 kB 6.1 MB/s eta 0:00:00
Collecting xformers (from axolotl==0.1)
Downloading xformers-0.0.20-cp310-cp310-manylinux2014_x86_64.whl (109.1 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 109.1/109.1 MB 8.6 MB/s eta 0:00:00
Collecting bert-score==0.3.13 (from axolotl==0.1)
Downloading bert_score-0.3.13-py3-none-any.whl (61 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 61.1/61.1 kB 8.2 MB/s eta 0:00:00
Collecting evaluate==0.4.0 (from axolotl==0.1)
Downloading evaluate-0.4.0-py3-none-any.whl (81 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 81.4/81.4 kB 11.0 MB/s eta 0:00:00
Collecting rouge-score==0.1.2 (from axolotl==0.1)
Downloading rouge_score-0.1.2.tar.gz (17 kB)
Preparing metadata (setup.py) ... done
Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (1.10.1)
Requirement already satisfied: scikit-learn==1.2.2 in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (1.2.2)
Requirement already satisfied: torch>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from bert-score==0.3.13->axolotl==0.1) (2.0.1+cu118)
Requirement already satisfied: pandas>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from bert-score==0.3.13->axolotl==0.1) (1.5.3)
Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from bert-score==0.3.13->axolotl==0.1) (1.22.4)
Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from bert-score==0.3.13->axolotl==0.1) (2.27.1)
Requirement already satisfied: tqdm>=4.31.1 in /usr/local/lib/python3.10/dist-packages (from bert-score==0.3.13->axolotl==0.1) (4.65.0)
Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (from bert-score==0.3.13->axolotl==0.1) (3.7.1)
Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from bert-score==0.3.13->axolotl==0.1) (23.1)
Collecting dill (from evaluate==0.4.0->axolotl==0.1)
Downloading dill-0.3.6-py3-none-any.whl (110 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 110.5/110.5 kB 15.1 MB/s eta 0:00:00
Collecting xxhash (from evaluate==0.4.0->axolotl==0.1)
Downloading xxhash-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (212 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 212.5/212.5 kB 22.1 MB/s eta 0:00:00
Collecting multiprocess (from evaluate==0.4.0->axolotl==0.1)
Downloading multiprocess-0.70.14-py310-none-any.whl (134 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 134.3/134.3 kB 16.4 MB/s eta 0:00:00
Requirement already satisfied: fsspec[http]>=2021.05.0 in /usr/local/lib/python3.10/dist-packages (from evaluate==0.4.0->axolotl==0.1) (2023.4.0)
Collecting huggingface-hub>=0.7.0 (from evaluate==0.4.0->axolotl==0.1)
Downloading huggingface_hub-0.15.1-py3-none-any.whl (236 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 236.8/236.8 kB 22.4 MB/s eta 0:00:00
Collecting responses<0.19 (from evaluate==0.4.0->axolotl==0.1)
Downloading responses-0.18.0-py3-none-any.whl (38 kB)
Requirement already satisfied: absl-py in /usr/local/lib/python3.10/dist-packages (from rouge-score==0.1.2->axolotl==0.1) (1.4.0)
Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from rouge-score==0.1.2->axolotl==0.1) (3.8.1)
Requirement already satisfied: six>=1.14.0 in /usr/local/lib/python3.10/dist-packages (from rouge-score==0.1.2->axolotl==0.1) (1.16.0)
Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn==1.2.2->axolotl==0.1) (1.2.0)
Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn==1.2.2->axolotl==0.1) (3.1.0)
Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate->axolotl==0.1) (5.9.5)
Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets->axolotl==0.1) (9.0.0)
Collecting aiohttp (from datasets->axolotl==0.1)
Downloading aiohttp-3.8.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.0 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.0/1.0 MB 56.1 MB/s eta 0:00:00
Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers@ git+https://github.com/huggingface/transformers.git->axolotl==0.1) (3.12.0)
Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers@ git+https://github.com/huggingface/transformers.git->axolotl==0.1) (2022.10.31)
Collecting tokenizers!=0.11.3,<0.14,>=0.11.1 (from transformers@ git+https://github.com/huggingface/transformers.git->axolotl==0.1)
Downloading tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 7.8/7.8 MB 115.6 MB/s eta 0:00:00
Collecting safetensors>=0.3.1 (from transformers@ git+https://github.com/huggingface/transformers.git->axolotl==0.1)
Downloading safetensors-0.3.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.3/1.3 MB 77.3 MB/s eta 0:00:00
Requirement already satisfied: termcolor in /usr/local/lib/python3.10/dist-packages (from fire->axolotl==0.1) (2.3.0)
Requirement already satisfied: Click!=8.0.0,>=7.0 in /usr/local/lib/python3.10/dist-packages (from wandb->axolotl==0.1) (8.1.3)
Collecting GitPython!=3.1.29,>=1.0.0 (from wandb->axolotl==0.1)
Downloading GitPython-3.1.31-py3-none-any.whl (184 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 184.3/184.3 kB 24.5 MB/s eta 0:00:00
Collecting sentry-sdk>=1.0.0 (from wandb->axolotl==0.1)
Downloading sentry_sdk-1.25.1-py2.py3-none-any.whl (206 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 206.7/206.7 kB 25.5 MB/s eta 0:00:00
Collecting docker-pycreds>=0.4.0 (from wandb->axolotl==0.1)
Downloading docker_pycreds-0.4.0-py2.py3-none-any.whl (9.0 kB)
Collecting pathtools (from wandb->axolotl==0.1)
Downloading pathtools-0.1.2.tar.gz (11 kB)
Preparing metadata (setup.py) ... done
Collecting setproctitle (from wandb->axolotl==0.1)
Downloading setproctitle-1.3.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (30 kB)
Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from wandb->axolotl==0.1) (67.7.2)
Requirement already satisfied: appdirs>=1.4.3 in /usr/local/lib/python3.10/dist-packages (from wandb->axolotl==0.1) (1.4.4)
Requirement already satisfied: protobuf!=4.21.0,<5,>=3.19.0 in /usr/local/lib/python3.10/dist-packages (from wandb->axolotl==0.1) (3.20.3)
Collecting pyre-extensions==0.0.29 (from xformers->axolotl==0.1)
Downloading pyre_extensions-0.0.29-py3-none-any.whl (12 kB)
Collecting typing-inspect (from pyre-extensions==0.0.29->xformers->axolotl==0.1)
Downloading typing_inspect-0.9.0-py3-none-any.whl (8.8 kB)
Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from pyre-extensions==0.0.29->xformers->axolotl==0.1) (4.5.0)
Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (1.11.1)
Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (3.1)
Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (3.1.2)
Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (2.0.0)
Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (3.25.2)
Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (16.0.5)
Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->axolotl==0.1) (23.1.0)
Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->axolotl==0.1) (2.0.12)
Collecting multidict<7.0,>=4.5 (from aiohttp->datasets->axolotl==0.1)
Downloading multidict-6.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (114 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 114.5/114.5 kB 15.1 MB/s eta 0:00:00
Collecting async-timeout<5.0,>=4.0.0a3 (from aiohttp->datasets->axolotl==0.1)
Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)
Collecting yarl<2.0,>=1.0 (from aiohttp->datasets->axolotl==0.1)
Downloading yarl-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (268 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 268.8/268.8 kB 32.4 MB/s eta 0:00:00
Collecting frozenlist>=1.1.1 (from aiohttp->datasets->axolotl==0.1)
Downloading frozenlist-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (149 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 149.6/149.6 kB 20.7 MB/s eta 0:00:00
Collecting aiosignal>=1.1.2 (from aiohttp->datasets->axolotl==0.1)
Downloading aiosignal-1.3.1-py3-none-any.whl (7.6 kB)
Collecting gitdb<5,>=4.0.1 (from GitPython!=3.1.29,>=1.0.0->wandb->axolotl==0.1)
Downloading gitdb-4.0.10-py3-none-any.whl (62 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 62.7/62.7 kB 9.6 MB/s eta 0:00:00
Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.0.1->bert-score==0.3.13->axolotl==0.1) (2.8.2)
Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.0.1->bert-score==0.3.13->axolotl==0.1) (2022.7.1)
Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->bert-score==0.3.13->axolotl==0.1) (1.26.15)
Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->bert-score==0.3.13->axolotl==0.1) (2022.12.7)
Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->bert-score==0.3.13->axolotl==0.1) (3.4)
Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->bert-score==0.3.13->axolotl==0.1) (1.0.7)
Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib->bert-score==0.3.13->axolotl==0.1) (0.11.0)
Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->bert-score==0.3.13->axolotl==0.1) (4.39.3)
Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->bert-score==0.3.13->axolotl==0.1) (1.4.4)
Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->bert-score==0.3.13->axolotl==0.1) (8.4.0)
Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->bert-score==0.3.13->axolotl==0.1) (3.0.9)
Collecting smmap<6,>=3.0.1 (from gitdb<5,>=4.0.1->GitPython!=3.1.29,>=1.0.0->wandb->axolotl==0.1)
Downloading smmap-5.0.0-py3-none-any.whl (24 kB)
Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (2.1.2)
Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (1.3.0)
Collecting mypy-extensions>=0.3.0 (from typing-inspect->pyre-extensions==0.0.29->xformers->axolotl==0.1)
Downloading mypy_extensions-1.0.0-py3-none-any.whl (4.7 kB)
Building wheels for collected packages: rouge-score, transformers, fire, pathtools
Building wheel for rouge-score (setup.py) ... done
Created wheel for rouge-score: filename=rouge_score-0.1.2-py3-none-any.whl size=24934 sha256=4f4dfa3de237b24376b3e253c0284d106034f2086de524eff974fec08528e009
Stored in directory: /root/.cache/pip/wheels/5f/dd/89/461065a73be61a532ff8599a28e9beef17985c9e9c31e541b4
Building wheel for transformers (pyproject.toml) ... done
Created wheel for transformers: filename=transformers-4.31.0.dev0-py3-none-any.whl size=7170473 sha256=164dfc7017620d58a037edc7544077dffe3b5500ed73f73567e8261748a2c55c
Stored in directory: /tmp/pip-ephem-wheel-cache-9jc0rsg4/wheels/e7/9c/5b/e1a9c8007c343041e61cc484433d512ea9274272e3fcbe7c16
Building wheel for fire (setup.py) ... done
Created wheel for fire: filename=fire-0.5.0-py2.py3-none-any.whl size=116932 sha256=2f59363ebbfbcbc1f0f4d0d6ebb5b0d101808f66163050b92f37574e1660821a
Stored in directory: /root/.cache/pip/wheels/90/d4/f7/9404e5db0116bd4d43e5666eaa3e70ab53723e1e3ea40c9a95
Building wheel for pathtools (setup.py) ... done
Created wheel for pathtools: filename=pathtools-0.1.2-py3-none-any.whl size=8791 sha256=dd6267f4f59e97d4e37762c1661146984b6d3dca2cd262a7ede8b09af42b0709
Stored in directory: /root/.cache/pip/wheels/e7/f3/22/152153d6eb222ee7a56ff8617d80ee5207207a8c00a7aab794
Successfully built rouge-score transformers fire pathtools
Installing collected packages: tokenizers, sentencepiece, safetensors, pathtools, bitsandbytes, addict, xxhash, smmap, setproctitle, sentry-sdk, mypy-extensions, multidict, frozenlist, fire, einops, docker-pycreds, dill, async-timeout, yarl, typing-inspect, rouge-score, responses, multiprocess, huggingface-hub, gitdb, aiosignal, transformers, pyre-extensions, GitPython, aiohttp, wandb, datasets, evaluate, xformers, bert-score, accelerate, axolotl
Running setup.py develop for axolotl
Successfully installed GitPython-3.1.31 accelerate-0.20.3 addict-2.4.0 aiohttp-3.8.4 aiosignal-1.3.1 async-timeout-4.0.2 axolotl-0.1 bert-score-0.3.13 bitsandbytes-0.39.0 datasets-2.12.0 dill-0.3.6 docker-pycreds-0.4.0 einops-0.6.1 evaluate-0.4.0 fire-0.5.0 frozenlist-1.3.3 gitdb-4.0.10 huggingface-hub-0.15.1 multidict-6.0.4 multiprocess-0.70.14 mypy-extensions-1.0.0 pathtools-0.1.2 pyre-extensions-0.0.29 responses-0.18.0 rouge-score-0.1.2 safetensors-0.3.1 sentencepiece-0.1.99 sentry-sdk-1.25.1 setproctitle-1.3.2 smmap-5.0.0 tokenizers-0.13.3 transformers-4.31.0.dev0 typing-inspect-0.9.0 wandb-0.15.4 xformers-0.0.20 xxhash-3.2.0 yarl-1.9.2
In [4]:
!pip install -U git+https://github.com/huggingface/peft.git
Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/ Collecting git+https://github.com/huggingface/peft.git Cloning https://github.com/huggingface/peft.git to /tmp/pip-req-build-vajurb24 Running command git clone --filter=blob:none --quiet https://github.com/huggingface/peft.git /tmp/pip-req-build-vajurb24 Resolved https://github.com/huggingface/peft.git to commit 189a6b8e357ecda05ccde13999e4c35759596a67 Installing build dependencies ... done Getting requirements to build wheel ... done Preparing metadata (pyproject.toml) ... done Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from peft==0.4.0.dev0) (1.22.4) Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from peft==0.4.0.dev0) (23.1) Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from peft==0.4.0.dev0) (5.9.5) Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from peft==0.4.0.dev0) (6.0) Requirement already satisfied: torch>=1.13.0 in /usr/local/lib/python3.10/dist-packages (from peft==0.4.0.dev0) (2.0.1+cu118) Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (from peft==0.4.0.dev0) (4.31.0.dev0) Requirement already satisfied: accelerate in /usr/local/lib/python3.10/dist-packages (from peft==0.4.0.dev0) (0.20.3) Requirement already satisfied: safetensors in /usr/local/lib/python3.10/dist-packages (from peft==0.4.0.dev0) (0.3.1) Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.0->peft==0.4.0.dev0) (3.12.0) Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.0->peft==0.4.0.dev0) (4.5.0) Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.0->peft==0.4.0.dev0) (1.11.1) Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.0->peft==0.4.0.dev0) (3.1) Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.0->peft==0.4.0.dev0) (3.1.2) Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.0->peft==0.4.0.dev0) (2.0.0) Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.13.0->peft==0.4.0.dev0) (3.25.2) Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.13.0->peft==0.4.0.dev0) (16.0.5) Requirement already satisfied: huggingface-hub<1.0,>=0.14.1 in /usr/local/lib/python3.10/dist-packages (from transformers->peft==0.4.0.dev0) (0.15.1) Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers->peft==0.4.0.dev0) (2022.10.31) Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers->peft==0.4.0.dev0) (2.27.1) Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.10/dist-packages (from transformers->peft==0.4.0.dev0) (0.13.3) Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers->peft==0.4.0.dev0) (4.65.0) Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers->peft==0.4.0.dev0) (2023.4.0) Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.13.0->peft==0.4.0.dev0) (2.1.2) Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers->peft==0.4.0.dev0) (1.26.15) Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers->peft==0.4.0.dev0) (2022.12.7) Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->transformers->peft==0.4.0.dev0) (2.0.12) Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers->peft==0.4.0.dev0) (3.4) Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.13.0->peft==0.4.0.dev0) (1.3.0) Building wheels for collected packages: peft Building wheel for peft (pyproject.toml) ... done Created wheel for peft: filename=peft-0.4.0.dev0-py3-none-any.whl size=59308 sha256=414b6cadbcf4223f6eb472b8145ff607431f62b70e86d9b45b7ecced17ddd36c Stored in directory: /tmp/pip-ephem-wheel-cache-5vh9a2_r/wheels/d7/c7/de/1368fac8590e1b103ddc2ec2a28ad51d83aded1a3830e8a087 Successfully built peft Installing collected packages: peft Successfully installed peft-0.4.0.dev0
In [8]:
!accelerate config --config_file configs/accelerate/default_config.yaml default
2023-06-10 11:16:08.926713: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT accelerate configuration saved at /root/.cache/huggingface/accelerate/default_config.yaml
In [18]:
%%writefile examples/falcon/config-1b-qlora.yml
# 1b: tiiuae/falcon-rw-1b
# 40b: tiiuae/falcon-40b
base_model: tiiuae/falcon-rw-1b
base_model_config: tiiuae/falcon-rw-1b
# required by falcon custom model code: https://huggingface.co/tiiuae/falcon-rw-1b/tree/main
trust_remote_code: true
model_type: AutoModelForCausalLM
tokenizer_type: AutoTokenizer
load_in_8bit: false
# enable 4bit for QLoRA
load_in_4bit: true
gptq: false
strict: false
push_dataset_to_hub:
datasets:
- path: QingyiSi/Alpaca-CoT
data_files:
- Chain-of-Thought/formatted_cot_data/gsm8k_train.json
type: "alpaca:chat"
dataset_prepared_path: last_run_prepared
val_set_size: 0.01
# enable QLoRA
adapter: qlora
lora_model_dir:
sequence_len: 2048
max_packed_sequence_len:
# hyperparameters from QLoRA paper Appendix B.2
# "We find hyperparameters to be largely robust across datasets"
lora_r: 64
lora_alpha: 16
# 0.1 for models up to 13B
# 0.05 for 33B and 65B models
lora_dropout: 0.05
# add LoRA modules on all linear layers of the base model
lora_target_modules:
lora_target_linear: true
lora_fan_in_fan_out:
wandb_project:
wandb_watch:
wandb_run_id:
wandb_log_model:
output_dir: ./qlora-out
some_config: some_string
# QLoRA paper Table 9
# - 16 for 7b & 13b
# - 32 for 33b, 64 for 64b
# Max size tested on A6000
# - 7b: 40
# - 40b: 4
# decrease if OOM, increase for max VRAM utilization
micro_batch_size: 1
gradient_accumulation_steps: 2
num_epochs: 3
# Optimizer for QLoRA
optimizer: paged_adamw_32bit
torchdistx_path:
lr_scheduler: cosine
# QLoRA paper Table 9
# - 2e-4 for 7b & 13b
# - 1e-4 for 33b & 64b
learning_rate: 0.0002
train_on_inputs: false
group_by_length: false
bf16: false
fp16: false
tf32: false
gradient_checkpointing: true
# stop training after this many evaluation losses have increased in a row
# https://huggingface.co/transformers/v4.2.2/_modules/transformers/trainer_callback.html#EarlyStoppingCallback
early_stopping_patience: 3
resume_from_checkpoint:
auto_resume_from_checkpoints: true
local_rank:
logging_steps: 1
xformers_attention: true
flash_attention:
gptq_groupsize:
gptq_model_v1:
warmup_steps: 10
eval_steps: 5
save_steps: 10
debug:
deepspeed:
weight_decay: 0.000001
fsdp:
fsdp_config:
special_tokens:
pad_token: "<|endoftext|>"
bos_token: ">>ABSTRACT<<"
eos_token: "<|endoftext|>"
Overwriting examples/falcon/config-1b-qlora.yml
In [24]:
%%writefile scripts/ft.py
from pathlib import Path
import fire
import finetune
from axolotl.utils.trainer import setup_trainer as setup_trainer_orig
def train_ex(
config: Path = Path("configs/"),
prepare_ds_only: bool = False,
**kwargs,
):
print('train_ex before')
finetune.train(config, prepare_ds_only, **kwargs)
print('train_ex after')
def setup_trainer_ex(cfg, train_dataset, eval_dataset, model, tokenizer):
print('setup_trainer_ex before')
print(cfg.some_config)
trainer = setup_trainer_orig(cfg, train_dataset, eval_dataset, model, tokenizer)
print('setup_trainer_ex after')
return trainer
finetune.setup_trainer = setup_trainer_ex
if __name__ == "__main__":
fire.Fire(train_ex)
Overwriting scripts/ft.py
In [ ]:
!accelerate launch scripts/ft.py examples/falcon/config-1b-qlora.yml
2023-06-10 11:45:48.232175: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
===================================BUG REPORT===================================
Welcome to bitsandbytes. For bug reports, please run
python -m bitsandbytes
and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues
================================================================================
bin /usr/local/lib/python3.10/dist-packages/bitsandbytes/libbitsandbytes_cuda118.so
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: /usr/lib64-nvidia did not contain ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] as expected! Searching further paths...
warn(msg)
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/sys/fs/cgroup/memory.events /var/colab/cgroup/jupyter-children/memory.events')}
warn(msg)
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('//172.28.0.1'), PosixPath('http'), PosixPath('8013')}
warn(msg)
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('--logtostderr --listen_host=172.28.0.12 --target_host=172.28.0.12 --tunnel_background_save_url=https'), PosixPath('//colab.research.google.com/tun/m/cc48301118ce562b961b3c22d803539adc1e0c19/gpu-t4-s-tccs3c5dv19r --tunnel_background_save_delay=10s --tunnel_periodic_background_save_frequency=30m0s --enable_output_coalescing=true --output_coalescing_required=true')}
warn(msg)
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/env/python')}
warn(msg)
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('module'), PosixPath('//ipykernel.pylab.backend_inline')}
warn(msg)
CUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching in backup paths...
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: Found duplicate ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] files: {PosixPath('/usr/local/cuda/lib64/libcudart.so'), PosixPath('/usr/local/cuda/lib64/libcudart.so.11.0')}.. We'll flip a coin and try one of these, in order to fail forward.
Either way, this might cause trouble in the future:
If you get `CUDA error: invalid device function` errors, the above might be the cause and the solution is to make sure only one ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] in the paths that we search based on your env.
warn(msg)
CUDA SETUP: CUDA runtime path found: /usr/local/cuda/lib64/libcudart.so
CUDA SETUP: Highest compute capability among GPUs detected: 7.5
CUDA SETUP: Detected CUDA version 118
CUDA SETUP: Loading binary /usr/local/lib/python3.10/dist-packages/bitsandbytes/libbitsandbytes_cuda118.so...
2023-06-10 11:45:56.336288: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
train_ex before
WARNING:root:`trust_remote_code` is set to true. Please make sure that you reviewed the remote code/model.
INFO:root:loading tokenizer... tiiuae/falcon-rw-1b
Using pad_token, but it is not set yet.
INFO:root:Loading prepared dataset from disk at last_run_prepared/0ecc5b78e3ce4254b22e749b093712b4...
INFO:root:Prepared dataset loaded from disk...
INFO:root:loading model and peft_config...
INFO:root:converting PEFT model w/ prepare_model_for_kbit_training
INFO:root:found linear modules: ['query_key_value', 'dense_h_to_4h', 'dense_4h_to_h', 'dense']
trainable params: 50,331,648 || all params: 757,911,552 || trainable%: 6.6408339953630895
setup_trainer_ex before
some_string
Using the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).
setup_trainer_ex after
INFO:root:Compiling torch model
INFO:root:Pre-saving adapter config to ./qlora-out
INFO:root:Starting trainer...
0% 0/11097 [00:00<?, ?it/s]You're using a GPT2TokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.
{'loss': 1.858, 'learning_rate': 2e-05, 'epoch': 0.0}
{'loss': 1.6529, 'learning_rate': 4e-05, 'epoch': 0.0}
{'loss': 1.6421, 'learning_rate': 6e-05, 'epoch': 0.0}
{'loss': 1.7999, 'learning_rate': 8e-05, 'epoch': 0.0}
{'loss': 1.7844, 'learning_rate': 0.0001, 'epoch': 0.0}
0% 5/11097 [00:07<4:34:41, 1.49s/it]
0% 0/75 [00:00<?, ?it/s]
3% 2/75 [00:00<00:05, 13.53it/s]
5% 4/75 [00:00<00:10, 6.96it/s]
7% 5/75 [00:00<00:12, 5.64it/s]
8% 6/75 [00:00<00:12, 5.69it/s]
9% 7/75 [00:01<00:13, 4.88it/s]
11% 8/75 [00:01<00:14, 4.59it/s]
12% 9/75 [00:01<00:12, 5.14it/s]
13% 10/75 [00:01<00:13, 4.93it/s]
15% 11/75 [00:02<00:12, 5.10it/s]
16% 12/75 [00:02<00:12, 4.98it/s]
17% 13/75 [00:02<00:12, 4.87it/s]
19% 14/75 [00:02<00:12, 4.87it/s]
20% 15/75 [00:02<00:12, 4.85it/s]
21% 16/75 [00:03<00:12, 4.56it/s]
23% 17/75 [00:03<00:12, 4.69it/s]
24% 18/75 [00:03<00:11, 4.95it/s]
25% 19/75 [00:03<00:11, 4.70it/s]
27% 20/75 [00:03<00:12, 4.53it/s]
28% 21/75 [00:04<00:11, 4.55it/s]
29% 22/75 [00:04<00:11, 4.46it/s]
31% 23/75 [00:04<00:12, 4.29it/s]
32% 24/75 [00:04<00:11, 4.40it/s]
33% 25/75 [00:05<00:10, 4.68it/s]
35% 26/75 [00:05<00:09, 4.99it/s]
36% 27/75 [00:05<00:09, 4.84it/s]
37% 28/75 [00:05<00:10, 4.63it/s]
39% 29/75 [00:05<00:10, 4.49it/s]
40% 30/75 [00:06<00:10, 4.27it/s]
41% 31/75 [00:06<00:10, 4.24it/s]
43% 32/75 [00:06<00:09, 4.56it/s]
44% 33/75 [00:06<00:09, 4.52it/s]
45% 34/75 [00:07<00:09, 4.36it/s]
47% 35/75 [00:07<00:09, 4.27it/s]
48% 36/75 [00:07<00:09, 4.29it/s]
49% 37/75 [00:07<00:08, 4.28it/s]
51% 38/75 [00:08<00:08, 4.30it/s]
52% 39/75 [00:08<00:08, 4.30it/s]
53% 40/75 [00:08<00:08, 4.36it/s]
55% 41/75 [00:08<00:07, 4.36it/s]
56% 42/75 [00:08<00:07, 4.33it/s]
57% 43/75 [00:09<00:07, 4.24it/s]
59% 44/75 [00:09<00:06, 4.69it/s]
60% 45/75 [00:09<00:06, 4.62it/s]
61% 46/75 [00:09<00:06, 4.23it/s]
63% 47/75 [00:10<00:07, 3.96it/s]
64% 48/75 [00:10<00:06, 3.98it/s]
65% 49/75 [00:10<00:06, 4.09it/s]
67% 50/75 [00:10<00:06, 3.96it/s]
68% 51/75 [00:11<00:05, 4.04it/s]
69% 52/75 [00:11<00:05, 4.38it/s]
71% 53/75 [00:11<00:05, 3.82it/s]
72% 54/75 [00:11<00:05, 3.73it/s]
73% 55/75 [00:12<00:04, 4.14it/s]
75% 56/75 [00:12<00:04, 4.23it/s]
76% 57/75 [00:12<00:04, 4.07it/s]
77% 58/75 [00:12<00:04, 4.20it/s]
79% 59/75 [00:13<00:03, 4.60it/s]
80% 60/75 [00:13<00:03, 4.33it/s]
81% 61/75 [00:13<00:03, 4.35it/s]
83% 62/75 [00:13<00:03, 4.09it/s]
84% 63/75 [00:14<00:02, 4.12it/s]
85% 64/75 [00:14<00:02, 4.34it/s]
87% 65/75 [00:14<00:02, 4.46it/s]
88% 66/75 [00:14<00:01, 4.82it/s]
89% 67/75 [00:14<00:01, 5.02it/s]
91% 68/75 [00:15<00:01, 4.76it/s]
92% 69/75 [00:15<00:01, 4.77it/s]
93% 70/75 [00:15<00:00, 5.02it/s]
95% 71/75 [00:15<00:00, 4.86it/s]
96% 72/75 [00:15<00:00, 4.79it/s]
97% 73/75 [00:16<00:00, 4.82it/s]
99% 74/75 [00:16<00:00, 4.51it/s]
{'eval_loss': 1.9294008016586304, 'eval_runtime': 16.7773, 'eval_samples_per_second': 4.47, 'eval_steps_per_second': 4.47, 'epoch': 0.0}
0% 5/11097 [00:24<4:34:41, 1.49s/it]
100% 75/75 [00:16<00:00, 4.46it/s]
{'loss': 2.3621, 'learning_rate': 0.00012, 'epoch': 0.0}
{'loss': 2.0084, 'learning_rate': 0.00014, 'epoch': 0.0}
{'loss': 1.4122, 'learning_rate': 0.00016, 'epoch': 0.0}
{'loss': 1.7174, 'learning_rate': 0.00018, 'epoch': 0.0}
{'loss': 1.4513, 'learning_rate': 0.0002, 'epoch': 0.0}
0% 10/11097 [00:31<8:12:26, 2.66s/it]
0% 0/75 [00:00<?, ?it/s]
3% 2/75 [00:00<00:05, 12.62it/s]
5% 4/75 [00:00<00:11, 6.21it/s]
7% 5/75 [00:00<00:13, 5.22it/s]
8% 6/75 [00:01<00:13, 5.27it/s]
9% 7/75 [00:01<00:14, 4.56it/s]
11% 8/75 [00:01<00:15, 4.27it/s]
12% 9/75 [00:01<00:14, 4.63it/s]
13% 10/75 [00:02<00:14, 4.47it/s]
15% 11/75 [00:02<00:13, 4.87it/s]
16% 12/75 [00:02<00:13, 4.71it/s]
17% 13/75 [00:02<00:13, 4.57it/s]
19% 14/75 [00:02<00:13, 4.50it/s]
20% 15/75 [00:03<00:12, 4.70it/s]
21% 16/75 [00:03<00:13, 4.29it/s]
23% 17/75 [00:03<00:13, 4.34it/s]
24% 18/75 [00:03<00:12, 4.72it/s]
25% 19/75 [00:03<00:12, 4.50it/s]
27% 20/75 [00:04<00:12, 4.41it/s]
28% 21/75 [00:04<00:12, 4.27it/s]
29% 22/75 [00:04<00:12, 4.32it/s]
31% 23/75 [00:04<00:12, 4.02it/s]
32% 24/75 [00:05<00:12, 4.14it/s]
33% 25/75 [00:05<00:11, 4.54it/s]
35% 26/75 [00:05<00:10, 4.81it/s]
36% 27/75 [00:05<00:10, 4.74it/s]
37% 28/75 [00:06<00:10, 4.63it/s]
39% 29/75 [00:06<00:10, 4.48it/s]
40% 30/75 [00:06<00:10, 4.22it/s]
41% 31/75 [00:06<00:10, 4.20it/s]
43% 32/75 [00:06<00:09, 4.44it/s]
44% 33/75 [00:07<00:09, 4.42it/s]
45% 34/75 [00:07<00:09, 4.25it/s]
47% 35/75 [00:07<00:09, 4.15it/s]
48% 36/75 [00:07<00:09, 4.15it/s]
49% 37/75 [00:08<00:09, 4.18it/s]
51% 38/75 [00:08<00:08, 4.20it/s]
52% 39/75 [00:08<00:08, 4.21it/s]
53% 40/75 [00:08<00:08, 4.23it/s]
55% 41/75 [00:09<00:07, 4.25it/s]
56% 42/75 [00:09<00:07, 4.23it/s]
57% 43/75 [00:09<00:07, 4.33it/s]
59% 44/75 [00:09<00:06, 4.58it/s]
60% 45/75 [00:10<00:06, 4.37it/s]
61% 46/75 [00:10<00:07, 4.06it/s]
63% 47/75 [00:10<00:07, 3.84it/s]
64% 48/75 [00:10<00:06, 3.86it/s]
65% 49/75 [00:11<00:06, 3.91it/s]
67% 50/75 [00:11<00:06, 3.86it/s]
68% 51/75 [00:11<00:05, 4.03it/s]
69% 52/75 [00:11<00:05, 4.18it/s]
71% 53/75 [00:12<00:05, 3.69it/s]
72% 54/75 [00:12<00:05, 3.74it/s]
73% 55/75 [00:12<00:04, 4.07it/s]
75% 56/75 [00:12<00:04, 4.02it/s]
76% 57/75 [00:13<00:04, 3.95it/s]
77% 58/75 [00:13<00:04, 4.13it/s]
79% 59/75 [00:13<00:03, 4.34it/s]
80% 60/75 [00:13<00:03, 4.17it/s]
81% 61/75 [00:14<00:03, 4.07it/s]
83% 62/75 [00:14<00:03, 3.91it/s]
84% 63/75 [00:14<00:03, 3.97it/s]
85% 64/75 [00:14<00:02, 4.05it/s]
87% 65/75 [00:15<00:02, 4.15it/s]
88% 66/75 [00:15<00:01, 4.60it/s]
89% 67/75 [00:15<00:01, 4.71it/s]
91% 68/75 [00:15<00:01, 4.50it/s]
92% 69/75 [00:15<00:01, 4.33it/s]
93% 70/75 [00:16<00:01, 4.70it/s]
95% 71/75 [00:16<00:00, 4.56it/s]
96% 72/75 [00:16<00:00, 4.48it/s]
97% 73/75 [00:16<00:00, 4.68it/s]
99% 74/75 [00:17<00:00, 4.32it/s]
{'eval_loss': 1.6967270374298096, 'eval_runtime': 17.4614, 'eval_samples_per_second': 4.295, 'eval_steps_per_second': 4.295, 'epoch': 0.0}
0% 10/11097 [00:48<8:12:26, 2.66s/it]
100% 75/75 [00:17<00:00, 4.27it/s]
{'loss': 1.5844, 'learning_rate': 0.00019999999598540582, 'epoch': 0.0}
{'loss': 1.3093, 'learning_rate': 0.00019999998394162357, 'epoch': 0.0}
{'loss': 2.4056, 'learning_rate': 0.00019999996386865424, 'epoch': 0.0}
{'loss': 1.8433, 'learning_rate': 0.0001999999357664994, 'epoch': 0.0}
{'loss': 2.3926, 'learning_rate': 0.00019999989963516136, 'epoch': 0.0}
0% 15/11097 [00:59<9:48:30, 3.19s/it]
0% 0/75 [00:00<?, ?it/s]
3% 2/75 [00:00<00:06, 11.57it/s]
5% 4/75 [00:00<00:11, 5.98it/s]
7% 5/75 [00:00<00:13, 5.15it/s]
8% 6/75 [00:01<00:13, 4.99it/s]
9% 7/75 [00:01<00:15, 4.42it/s]
11% 8/75 [00:01<00:16, 4.08it/s]
12% 9/75 [00:01<00:14, 4.43it/s]
13% 10/75 [00:02<00:14, 4.44it/s]
15% 11/75 [00:02<00:13, 4.66it/s]
16% 12/75 [00:02<00:13, 4.52it/s]
17% 13/75 [00:02<00:14, 4.35it/s]
19% 14/75 [00:02<00:14, 4.34it/s]
20% 15/75 [00:03<00:13, 4.46it/s]
21% 16/75 [00:03<00:14, 4.12it/s]
23% 17/75 [00:03<00:14, 4.14it/s]
24% 18/75 [00:03<00:12, 4.42it/s]
25% 19/75 [00:04<00:12, 4.31it/s]
27% 20/75 [00:04<00:13, 4.21it/s]
28% 21/75 [00:04<00:13, 4.09it/s]
29% 22/75 [00:04<00:13, 4.04it/s]
31% 23/75 [00:05<00:13, 3.87it/s]
32% 24/75 [00:05<00:13, 3.92it/s]
33% 25/75 [00:05<00:11, 4.24it/s]
35% 26/75 [00:05<00:10, 4.54it/s]
36% 27/75 [00:06<00:11, 4.33it/s]
37% 28/75 [00:06<00:11, 4.23it/s]
39% 29/75 [00:06<00:10, 4.18it/s]
40% 30/75 [00:06<00:11, 3.91it/s]
41% 31/75 [00:07<00:11, 3.74it/s]
43% 32/75 [00:07<00:10, 4.17it/s]
44% 33/75 [00:07<00:10, 4.17it/s]
45% 34/75 [00:07<00:10, 4.04it/s]
47% 35/75 [00:08<00:10, 3.82it/s]
48% 36/75 [00:08<00:10, 3.88it/s]
49% 37/75 [00:08<00:09, 3.88it/s]
51% 38/75 [00:08<00:09, 4.00it/s]
In [20]:
!rm -rf ./qlora-out
In [ ]: