1. Get Axolotl and install its dependencies¶
In [1]:
!git clone https://github.com/OpenAccess-AI-Collective/axolotl
fatal: destination path 'axolotl' already exists and is not an empty directory.
In [2]:
%cd axolotl
/content/axolotl
In [3]:
!pip3 install -e .
Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/
Obtaining file:///content/axolotl
Preparing metadata (setup.py) ... done
Collecting transformers@ git+https://github.com/huggingface/transformers.git (from axolotl==0.1)
Cloning https://github.com/huggingface/transformers.git to /tmp/pip-install-9s5i8p5v/transformers_9f5cd6dfc56647f2af7ed8386a731861
Running command git clone --filter=blob:none --quiet https://github.com/huggingface/transformers.git /tmp/pip-install-9s5i8p5v/transformers_9f5cd6dfc56647f2af7ed8386a731861
Resolved https://github.com/huggingface/transformers.git to commit 0b7b4429c78de68acaf72224eb6dae43616d820c
Installing build dependencies ... done
Getting requirements to build wheel ... done
Preparing metadata (pyproject.toml) ... done
Requirement already satisfied: bitsandbytes>=0.39.0 in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (0.39.0)
Requirement already satisfied: accelerate in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (0.20.3)
Requirement already satisfied: addict in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (2.4.0)
Requirement already satisfied: fire in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (0.5.0)
Requirement already satisfied: PyYAML==6.0 in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (6.0)
Requirement already satisfied: datasets in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (2.13.0)
Requirement already satisfied: sentencepiece in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (0.1.99)
Requirement already satisfied: wandb in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (0.15.4)
Requirement already satisfied: einops in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (0.6.1)
Requirement already satisfied: xformers in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (0.0.20)
Requirement already satisfied: optimum in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (1.8.6)
Requirement already satisfied: bert-score==0.3.13 in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (0.3.13)
Requirement already satisfied: evaluate==0.4.0 in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (0.4.0)
Requirement already satisfied: rouge-score==0.1.2 in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (0.1.2)
Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (1.10.1)
Requirement already satisfied: scikit-learn==1.2.2 in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (1.2.2)
Requirement already satisfied: torch>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from bert-score==0.3.13->axolotl==0.1) (2.0.1+cu118)
Requirement already satisfied: pandas>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from bert-score==0.3.13->axolotl==0.1) (1.5.3)
Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from bert-score==0.3.13->axolotl==0.1) (1.22.4)
Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from bert-score==0.3.13->axolotl==0.1) (2.27.1)
Requirement already satisfied: tqdm>=4.31.1 in /usr/local/lib/python3.10/dist-packages (from bert-score==0.3.13->axolotl==0.1) (4.65.0)
Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (from bert-score==0.3.13->axolotl==0.1) (3.7.1)
Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from bert-score==0.3.13->axolotl==0.1) (23.1)
Requirement already satisfied: dill in /usr/local/lib/python3.10/dist-packages (from evaluate==0.4.0->axolotl==0.1) (0.3.6)
Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from evaluate==0.4.0->axolotl==0.1) (3.2.0)
Requirement already satisfied: multiprocess in /usr/local/lib/python3.10/dist-packages (from evaluate==0.4.0->axolotl==0.1) (0.70.14)
Requirement already satisfied: fsspec[http]>=2021.05.0 in /usr/local/lib/python3.10/dist-packages (from evaluate==0.4.0->axolotl==0.1) (2023.4.0)
Requirement already satisfied: huggingface-hub>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from evaluate==0.4.0->axolotl==0.1) (0.15.1)
Requirement already satisfied: responses<0.19 in /usr/local/lib/python3.10/dist-packages (from evaluate==0.4.0->axolotl==0.1) (0.18.0)
Requirement already satisfied: absl-py in /usr/local/lib/python3.10/dist-packages (from rouge-score==0.1.2->axolotl==0.1) (1.4.0)
Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from rouge-score==0.1.2->axolotl==0.1) (3.8.1)
Requirement already satisfied: six>=1.14.0 in /usr/local/lib/python3.10/dist-packages (from rouge-score==0.1.2->axolotl==0.1) (1.16.0)
Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn==1.2.2->axolotl==0.1) (1.2.0)
Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn==1.2.2->axolotl==0.1) (3.1.0)
Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate->axolotl==0.1) (5.9.5)
Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets->axolotl==0.1) (9.0.0)
Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets->axolotl==0.1) (3.8.4)
Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers@ git+https://github.com/huggingface/transformers.git->axolotl==0.1) (3.12.0)
Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers@ git+https://github.com/huggingface/transformers.git->axolotl==0.1) (2022.10.31)
Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.10/dist-packages (from transformers@ git+https://github.com/huggingface/transformers.git->axolotl==0.1) (0.13.3)
Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers@ git+https://github.com/huggingface/transformers.git->axolotl==0.1) (0.3.1)
Requirement already satisfied: termcolor in /usr/local/lib/python3.10/dist-packages (from fire->axolotl==0.1) (2.3.0)
Requirement already satisfied: coloredlogs in /usr/local/lib/python3.10/dist-packages (from optimum->axolotl==0.1) (15.0.1)
Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from optimum->axolotl==0.1) (1.11.1)
Requirement already satisfied: torchvision in /usr/local/lib/python3.10/dist-packages (from optimum->axolotl==0.1) (0.15.2+cu118)
Requirement already satisfied: Click!=8.0.0,>=7.0 in /usr/local/lib/python3.10/dist-packages (from wandb->axolotl==0.1) (8.1.3)
Requirement already satisfied: GitPython!=3.1.29,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb->axolotl==0.1) (3.1.31)
Requirement already satisfied: sentry-sdk>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb->axolotl==0.1) (1.25.1)
Requirement already satisfied: docker-pycreds>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from wandb->axolotl==0.1) (0.4.0)
Requirement already satisfied: pathtools in /usr/local/lib/python3.10/dist-packages (from wandb->axolotl==0.1) (0.1.2)
Requirement already satisfied: setproctitle in /usr/local/lib/python3.10/dist-packages (from wandb->axolotl==0.1) (1.3.2)
Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from wandb->axolotl==0.1) (67.7.2)
Requirement already satisfied: appdirs>=1.4.3 in /usr/local/lib/python3.10/dist-packages (from wandb->axolotl==0.1) (1.4.4)
Requirement already satisfied: protobuf!=4.21.0,<5,>=3.19.0 in /usr/local/lib/python3.10/dist-packages (from wandb->axolotl==0.1) (3.20.3)
Requirement already satisfied: pyre-extensions==0.0.29 in /usr/local/lib/python3.10/dist-packages (from xformers->axolotl==0.1) (0.0.29)
Requirement already satisfied: typing-inspect in /usr/local/lib/python3.10/dist-packages (from pyre-extensions==0.0.29->xformers->axolotl==0.1) (0.9.0)
Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from pyre-extensions==0.0.29->xformers->axolotl==0.1) (4.5.0)
Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (3.1)
Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (3.1.2)
Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (2.0.0)
Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (3.25.2)
Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (16.0.5)
Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->axolotl==0.1) (23.1.0)
Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->axolotl==0.1) (2.0.12)
Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->axolotl==0.1) (6.0.4)
Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->axolotl==0.1) (4.0.2)
Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->axolotl==0.1) (1.9.2)
Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->axolotl==0.1) (1.3.3)
Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->axolotl==0.1) (1.3.1)
Requirement already satisfied: gitdb<5,>=4.0.1 in /usr/local/lib/python3.10/dist-packages (from GitPython!=3.1.29,>=1.0.0->wandb->axolotl==0.1) (4.0.10)
Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.0.1->bert-score==0.3.13->axolotl==0.1) (2.8.2)
Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.0.1->bert-score==0.3.13->axolotl==0.1) (2022.7.1)
Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->bert-score==0.3.13->axolotl==0.1) (1.26.15)
Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->bert-score==0.3.13->axolotl==0.1) (2022.12.7)
Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->bert-score==0.3.13->axolotl==0.1) (3.4)
Requirement already satisfied: humanfriendly>=9.1 in /usr/local/lib/python3.10/dist-packages (from coloredlogs->optimum->axolotl==0.1) (10.0)
Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->bert-score==0.3.13->axolotl==0.1) (1.0.7)
Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib->bert-score==0.3.13->axolotl==0.1) (0.11.0)
Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->bert-score==0.3.13->axolotl==0.1) (4.39.3)
Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->bert-score==0.3.13->axolotl==0.1) (1.4.4)
Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->bert-score==0.3.13->axolotl==0.1) (8.4.0)
Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->bert-score==0.3.13->axolotl==0.1) (3.0.9)
Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->optimum->axolotl==0.1) (1.3.0)
Requirement already satisfied: smmap<6,>=3.0.1 in /usr/local/lib/python3.10/dist-packages (from gitdb<5,>=4.0.1->GitPython!=3.1.29,>=1.0.0->wandb->axolotl==0.1) (5.0.0)
Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (2.1.2)
Requirement already satisfied: mypy-extensions>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from typing-inspect->pyre-extensions==0.0.29->xformers->axolotl==0.1) (1.0.0)
Installing collected packages: axolotl
Attempting uninstall: axolotl
Found existing installation: axolotl 0.1
Uninstalling axolotl-0.1:
Successfully uninstalled axolotl-0.1
Running setup.py develop for axolotl
Successfully installed axolotl-0.1
2. Generate default config for accelerate¶
In [4]:
!accelerate config --config_file configs/accelerate/default_config.yaml default
2023-06-16 03:41:48.444526: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT Configuration already exists at /root/.cache/huggingface/accelerate/default_config.yaml, will not override. Run `accelerate config` manually or pass a different `save_location`.
You can read the comments in the config and tweak it.
3. Install QLoRA dependencies¶
In [5]:
!pip install git+https://github.com/huggingface/peft.git
Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/ Collecting git+https://github.com/huggingface/peft.git Cloning https://github.com/huggingface/peft.git to /tmp/pip-req-build-mt86vd8z Running command git clone --filter=blob:none --quiet https://github.com/huggingface/peft.git /tmp/pip-req-build-mt86vd8z Resolved https://github.com/huggingface/peft.git to commit 9320373c129f71298a181989af1b134b138d2d43 Installing build dependencies ... done Getting requirements to build wheel ... done Preparing metadata (pyproject.toml) ... done Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from peft==0.4.0.dev0) (1.22.4) Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from peft==0.4.0.dev0) (23.1) Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from peft==0.4.0.dev0) (5.9.5) Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from peft==0.4.0.dev0) (6.0) Requirement already satisfied: torch>=1.13.0 in /usr/local/lib/python3.10/dist-packages (from peft==0.4.0.dev0) (2.0.1+cu118) Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (from peft==0.4.0.dev0) (4.31.0.dev0) Requirement already satisfied: accelerate in /usr/local/lib/python3.10/dist-packages (from peft==0.4.0.dev0) (0.20.3) Requirement already satisfied: safetensors in /usr/local/lib/python3.10/dist-packages (from peft==0.4.0.dev0) (0.3.1) Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.0->peft==0.4.0.dev0) (3.12.0) Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.0->peft==0.4.0.dev0) (4.5.0) Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.0->peft==0.4.0.dev0) (1.11.1) Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.0->peft==0.4.0.dev0) (3.1) Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.0->peft==0.4.0.dev0) (3.1.2) Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.0->peft==0.4.0.dev0) (2.0.0) Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.13.0->peft==0.4.0.dev0) (3.25.2) Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.13.0->peft==0.4.0.dev0) (16.0.5) Requirement already satisfied: huggingface-hub<1.0,>=0.14.1 in /usr/local/lib/python3.10/dist-packages (from transformers->peft==0.4.0.dev0) (0.15.1) Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers->peft==0.4.0.dev0) (2022.10.31) Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers->peft==0.4.0.dev0) (2.27.1) Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.10/dist-packages (from transformers->peft==0.4.0.dev0) (0.13.3) Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers->peft==0.4.0.dev0) (4.65.0) Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers->peft==0.4.0.dev0) (2023.4.0) Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.13.0->peft==0.4.0.dev0) (2.1.2) Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers->peft==0.4.0.dev0) (1.26.15) Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers->peft==0.4.0.dev0) (2022.12.7) Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->transformers->peft==0.4.0.dev0) (2.0.12) Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers->peft==0.4.0.dev0) (3.4) Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.13.0->peft==0.4.0.dev0) (1.3.0)
4. Set W&B to offline mode¶
In [6]:
%env WANDB_MODE=offline
env: WANDB_MODE=offline
This is to skip some extra setup steps, you can also choose to login to your W&B account before training.
5. Start training and enjoy!¶
❌ flash_optimum + qlora + falcon¶
In [8]:
%%writefile falcon-1b-qlora.yml
# 1b: tiiuae/falcon-rw-1b
# 40b: tiiuae/falcon-40b
base_model: tiiuae/falcon-rw-1b
base_model_config: tiiuae/falcon-rw-1b
# required by falcon custom model code: https://huggingface.co/tiiuae/falcon-rw-1b/tree/main
trust_remote_code: true
model_type: AutoModelForCausalLM
tokenizer_type: AutoTokenizer
load_in_8bit: false
# enable 4bit for QLoRA
load_in_4bit: true
gptq: false
strict: false
push_dataset_to_hub:
datasets:
- path: QingyiSi/Alpaca-CoT
data_files:
- Chain-of-Thought/formatted_cot_data/gsm8k_train.json
type: "alpaca:chat"
dataset_prepared_path: last_run_prepared
val_set_size: 0.01
# enable QLoRA
adapter: qlora
lora_model_dir:
sequence_len: 2048
max_packed_sequence_len:
# hyperparameters from QLoRA paper Appendix B.2
# "We find hyperparameters to be largely robust across datasets"
lora_r: 64
lora_alpha: 16
# 0.1 for models up to 13B
# 0.05 for 33B and 65B models
lora_dropout: 0.05
# add LoRA modules on all linear layers of the base model
lora_target_modules:
lora_target_linear: true
lora_fan_in_fan_out:
wandb_project:
wandb_watch:
wandb_run_id:
wandb_log_model:
output_dir: ./qlora-out
# QLoRA paper Table 9
# - 16 for 7b & 13b
# - 32 for 33b, 64 for 64b
# Max size tested on A6000
# - 7b: 40
# - 40b: 4
# decrease if OOM, increase for max VRAM utilization
micro_batch_size: 1
gradient_accumulation_steps: 2
num_epochs: 3
# Optimizer for QLoRA
optimizer: paged_adamw_32bit
torchdistx_path:
lr_scheduler: cosine
# QLoRA paper Table 9
# - 2e-4 for 7b & 13b
# - 1e-4 for 33b & 64b
learning_rate: 0.0002
train_on_inputs: false
group_by_length: false
bf16: false
fp16: false
tf32: false
gradient_checkpointing: true
# stop training after this many evaluation losses have increased in a row
# https://huggingface.co/transformers/v4.2.2/_modules/transformers/trainer_callback.html#EarlyStoppingCallback
# early_stopping_patience: 3
resume_from_checkpoint:
auto_resume_from_checkpoints: true
local_rank:
logging_steps: 1
xformers_attention:
flash_attention:
flash_optimum: true
gptq_groupsize:
gptq_model_v1:
warmup_steps: 10
eval_steps: 5
save_steps: 10
debug:
deepspeed:
weight_decay: 0.000001
fsdp:
fsdp_config:
special_tokens:
pad_token: "<|endoftext|>"
bos_token: ">>ABSTRACT<<"
eos_token: "<|endoftext|>"
Overwriting falcon-1b-qlora.yml
In [9]:
!accelerate launch scripts/finetune.py falcon-1b-qlora.yml
2023-06-16 03:52:45.471547: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
2023-06-16 03:52:52.770823: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
===================================BUG REPORT===================================
Welcome to bitsandbytes. For bug reports, please run
python -m bitsandbytes
and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues
================================================================================
bin /usr/local/lib/python3.10/dist-packages/bitsandbytes/libbitsandbytes_cuda118.so
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: /usr/lib64-nvidia did not contain ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] as expected! Searching further paths...
warn(msg)
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/sys/fs/cgroup/memory.events /var/colab/cgroup/jupyter-children/memory.events')}
warn(msg)
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('http'), PosixPath('//172.28.0.1'), PosixPath('8013')}
warn(msg)
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('--logtostderr --listen_host=172.28.0.12 --target_host=172.28.0.12 --tunnel_background_save_url=https'), PosixPath('//colab.research.google.com/tun/m/cc48301118ce562b961b3c22d803539adc1e0c19/gpu-t4-s-19a78udzj535v --tunnel_background_save_delay=10s --tunnel_periodic_background_save_frequency=30m0s --enable_output_coalescing=true --output_coalescing_required=true')}
warn(msg)
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/env/python')}
warn(msg)
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('module'), PosixPath('//ipykernel.pylab.backend_inline')}
warn(msg)
CUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching in backup paths...
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: Found duplicate ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] files: {PosixPath('/usr/local/cuda/lib64/libcudart.so.11.0'), PosixPath('/usr/local/cuda/lib64/libcudart.so')}.. We'll flip a coin and try one of these, in order to fail forward.
Either way, this might cause trouble in the future:
If you get `CUDA error: invalid device function` errors, the above might be the cause and the solution is to make sure only one ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] in the paths that we search based on your env.
warn(msg)
CUDA SETUP: CUDA runtime path found: /usr/local/cuda/lib64/libcudart.so.11.0
CUDA SETUP: Highest compute capability among GPUs detected: 7.5
CUDA SETUP: Detected CUDA version 118
CUDA SETUP: Loading binary /usr/local/lib/python3.10/dist-packages/bitsandbytes/libbitsandbytes_cuda118.so...
WARNING:root:`trust_remote_code` is set to true. Please make sure that you reviewed the remote code/model.
WARNING:root:BetterTransformers probably doesn't work with PEFT adapters
WARNING:root:You should probably set bfloat16 or float16 to true to load the model in float16 for BetterTransformers
INFO:root:loading tokenizer... tiiuae/falcon-rw-1b
Using pad_token, but it is not set yet.
INFO:root:Loading prepared dataset from disk at last_run_prepared/0ecc5b78e3ce4254b22e749b093712b4...
INFO:root:Prepared dataset loaded from disk...
INFO:root:loading model and peft_config...
INFO:root:converting PEFT model w/ prepare_model_for_kbit_training
INFO:root:found linear modules: ['dense', 'dense_4h_to_h', 'dense_h_to_4h', 'query_key_value']
trainable params: 50,331,648 || all params: 757,911,552 || trainable%: 6.6408339953630895
╭───────────────────── Traceback (most recent call last) ──────────────────────╮
│ /content/axolotl/scripts/finetune.py:352 in <module> │
│ │
│ 349 │
│ 350 │
│ 351 if __name__ == "__main__": │
│ ❱ 352 │ fire.Fire(train) │
│ 353 │
│ │
│ /usr/local/lib/python3.10/dist-packages/fire/core.py:141 in Fire │
│ │
│ 138 │ context.update(caller_globals) │
│ 139 │ context.update(caller_locals) │
│ 140 │
│ ❱ 141 component_trace = _Fire(component, args, parsed_flag_args, context, │
│ 142 │
│ 143 if component_trace.HasError(): │
│ 144 │ _DisplayError(component_trace) │
│ │
│ /usr/local/lib/python3.10/dist-packages/fire/core.py:475 in _Fire │
│ │
│ 472 │ is_class = inspect.isclass(component) │
│ 473 │ │
│ 474 │ try: │
│ ❱ 475 │ │ component, remaining_args = _CallAndUpdateTrace( │
│ 476 │ │ │ component, │
│ 477 │ │ │ remaining_args, │
│ 478 │ │ │ component_trace, │
│ │
│ /usr/local/lib/python3.10/dist-packages/fire/core.py:691 in │
│ _CallAndUpdateTrace │
│ │
│ 688 │ loop = asyncio.get_event_loop() │
│ 689 │ component = loop.run_until_complete(fn(*varargs, **kwargs)) │
│ 690 else: │
│ ❱ 691 │ component = fn(*varargs, **kwargs) │
│ 692 │
│ 693 if treatment == 'class': │
│ 694 │ action = trace.INSTANTIATED_CLASS │
│ │
│ /content/axolotl/scripts/finetune.py:251 in train │
│ │
│ 248 │ │
│ 249 │ # Load the model and tokenizer │
│ 250 │ logging.info("loading model and peft_config...") │
│ ❱ 251 │ model, peft_config = load_model( │
│ 252 │ │ cfg.base_model, │
│ 253 │ │ cfg.base_model_config, │
│ 254 │ │ cfg.model_type, │
│ │
│ /content/axolotl/src/axolotl/utils/models.py:354 in load_model │
│ │
│ 351 │ model.config.use_cache = False │
│ 352 │ │
│ 353 │ if cfg.flash_optimum: │
│ ❱ 354 │ │ model = BetterTransformer.transform(model) │
│ 355 │ │
│ 356 │ # TODO resume_from_checkpoint handling │
│ 357 │ return model, lora_config │
│ │
│ /usr/lib/python3.10/contextlib.py:79 in inner │
│ │
│ 76 │ │ @wraps(func) │
│ 77 │ │ def inner(*args, **kwds): │
│ 78 │ │ │ with self._recreate_cm(): │
│ ❱ 79 │ │ │ │ return func(*args, **kwds) │
│ 80 │ │ return inner │
│ 81 │
│ 82 │
│ │
│ /usr/local/lib/python3.10/dist-packages/optimum/bettertransformer/transforma │
│ tion.py:211 in transform │
│ │
│ 208 │ │ │ │ f" {BetterTransformerManager.MODEL_MAPPING.keys()}." │
│ 209 │ │ │ ) │
│ 210 │ │ if not BetterTransformerManager.supports(model.config.model_ty │
│ ❱ 211 │ │ │ raise NotImplementedError( │
│ 212 │ │ │ │ f"The model type {model.config.model_type} is not yet │
│ 213 │ │ │ │ f" to open an issue at https://github.com/huggingface/ │
│ 214 │ │ │ │ f" Currently supported models are: {BetterTransformerM │
╰──────────────────────────────────────────────────────────────────────────────╯
NotImplementedError: The model type RefinedWebModel is not yet supported to be
used with BetterTransformer. Feel free to open an issue at
https://github.com/huggingface/optimum/issues if you would like this model type
to be supported. Currently supported models are: dict_keys(['albert', 'bart',
'bert', 'bert-generation', 'blenderbot', 'camembert', 'clip', 'codegen',
'data2vec-text', 'deit', 'distilbert', 'electra', 'ernie', 'fsmt', 'gpt2',
'gptj', 'gpt_neo', 'gpt_neox', 'hubert', 'layoutlm', 'm2m_100', 'marian',
'markuplm', 'mbart', 'opt', 'pegasus', 'rembert', 'prophetnet', 'roberta',
'roc_bert', 'roformer', 'splinter', 'tapas', 't5', 'vilt', 'vit', 'vit_mae',
'vit_msn', 'wav2vec2', 'whisper', 'xlm-roberta', 'yolos']).
╭───────────────────── Traceback (most recent call last) ──────────────────────╮
│ /usr/local/bin/accelerate:8 in <module> │
│ │
│ 5 from accelerate.commands.accelerate_cli import main │
│ 6 if __name__ == '__main__': │
│ 7 │ sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) │
│ ❱ 8 │ sys.exit(main()) │
│ 9 │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/commands/accelerate_cli.p │
│ y:45 in main │
│ │
│ 42 │ │ exit(1) │
│ 43 │ │
│ 44 │ # Run │
│ ❱ 45 │ args.func(args) │
│ 46 │
│ 47 │
│ 48 if __name__ == "__main__": │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/commands/launch.py:941 in │
│ launch_command │
│ │
│ 938 │ elif defaults is not None and defaults.compute_environment == Comp │
│ 939 │ │ sagemaker_launcher(defaults, args) │
│ 940 │ else: │
│ ❱ 941 │ │ simple_launcher(args) │
│ 942 │
│ 943 │
│ 944 def main(): │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/commands/launch.py:603 in │
│ simple_launcher │
│ │
│ 600 │ process.wait() │
│ 601 │ if process.returncode != 0: │
│ 602 │ │ if not args.quiet: │
│ ❱ 603 │ │ │ raise subprocess.CalledProcessError(returncode=process.ret │
│ 604 │ │ else: │
│ 605 │ │ │ sys.exit(1) │
│ 606 │
╰──────────────────────────────────────────────────────────────────────────────╯
CalledProcessError: Command '['/usr/bin/python3', 'scripts/finetune.py',
'falcon-1b-qlora.yml']' returned non-zero exit status 1.
❌ flash_optimum + qlora + pythia¶
In [10]:
%%writefile pythia-1b-qlora.yml
base_model: EleutherAI/pythia-1b-deduped
base_model_config: EleutherAI/pythia-1b-deduped
base_model_ignore_patterns: pytorch* # prefer safetensors
model_type: GPTNeoXForCausalLM
tokenizer_type: AutoTokenizer
load_in_8bit: false
# enable 4bit for QLoRA
load_in_4bit: true
gptq: false
strict: false
push_dataset_to_hub:
datasets:
- path: QingyiSi/Alpaca-CoT
data_files:
- Chain-of-Thought/formatted_cot_data/gsm8k_train.json
type: "alpaca:chat"
dataset_prepared_path: last_run_prepared
val_set_size: 0.01
# enable QLoRA
adapter: qlora
lora_model_dir:
sequence_len: 2048
max_packed_sequence_len:
# hyperparameters from QLoRA paper Appendix B.2
# "We find hyperparameters to be largely robust across datasets"
lora_r: 64
lora_alpha: 16
# 0.1 for models up to 13B
# 0.05 for 33B and 65B models
lora_dropout: 0.05
# add LoRA modules on all linear layers of the base model
lora_target_modules:
lora_target_linear: true
lora_fan_in_fan_out:
wandb_project:
wandb_watch:
wandb_run_id:
wandb_log_model:
output_dir: ./qlora-out
# QLoRA paper Table 9
# - 16 for 7b & 13b
# - 32 for 33b, 64 for 64b
# Max size tested on A6000
# - 7b: 40
# - 40b: 4
# decrease if OOM, increase for max VRAM utilization
micro_batch_size: 1
gradient_accumulation_steps: 2
num_epochs: 3
# Optimizer for QLoRA
optimizer: paged_adamw_32bit
torchdistx_path:
lr_scheduler: cosine
# QLoRA paper Table 9
# - 2e-4 for 7b & 13b
# - 1e-4 for 33b & 64b
learning_rate: 0.0002
train_on_inputs: false
group_by_length: false
bf16: false
fp16: false
tf32: false
gradient_checkpointing: true
# stop training after this many evaluation losses have increased in a row
# https://huggingface.co/transformers/v4.2.2/_modules/transformers/trainer_callback.html#EarlyStoppingCallback
# early_stopping_patience: 3
resume_from_checkpoint:
auto_resume_from_checkpoints: true
local_rank:
logging_steps: 1
xformers_attention:
flash_attention:
flash_optimum: true
gptq_groupsize:
gptq_model_v1:
warmup_steps: 10
eval_steps: 5
save_steps: 10
debug:
deepspeed:
weight_decay: 0.000001
fsdp:
fsdp_config:
special_tokens:
pad_token: "<|endoftext|>"
bos_token: ">>ABSTRACT<<"
eos_token: "<|endoftext|>"
Writing pythia-1b-qlora.yml
In [11]:
!accelerate launch scripts/finetune.py pythia-1b-qlora.yml
2023-06-16 03:53:43.143557: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
2023-06-16 03:53:49.889816: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
===================================BUG REPORT===================================
Welcome to bitsandbytes. For bug reports, please run
python -m bitsandbytes
and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues
================================================================================
bin /usr/local/lib/python3.10/dist-packages/bitsandbytes/libbitsandbytes_cuda118.so
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: /usr/lib64-nvidia did not contain ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] as expected! Searching further paths...
warn(msg)
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/sys/fs/cgroup/memory.events /var/colab/cgroup/jupyter-children/memory.events')}
warn(msg)
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('8013'), PosixPath('http'), PosixPath('//172.28.0.1')}
warn(msg)
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('--logtostderr --listen_host=172.28.0.12 --target_host=172.28.0.12 --tunnel_background_save_url=https'), PosixPath('//colab.research.google.com/tun/m/cc48301118ce562b961b3c22d803539adc1e0c19/gpu-t4-s-19a78udzj535v --tunnel_background_save_delay=10s --tunnel_periodic_background_save_frequency=30m0s --enable_output_coalescing=true --output_coalescing_required=true')}
warn(msg)
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/env/python')}
warn(msg)
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('module'), PosixPath('//ipykernel.pylab.backend_inline')}
warn(msg)
CUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching in backup paths...
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: Found duplicate ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] files: {PosixPath('/usr/local/cuda/lib64/libcudart.so.11.0'), PosixPath('/usr/local/cuda/lib64/libcudart.so')}.. We'll flip a coin and try one of these, in order to fail forward.
Either way, this might cause trouble in the future:
If you get `CUDA error: invalid device function` errors, the above might be the cause and the solution is to make sure only one ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] in the paths that we search based on your env.
warn(msg)
CUDA SETUP: CUDA runtime path found: /usr/local/cuda/lib64/libcudart.so.11.0
CUDA SETUP: Highest compute capability among GPUs detected: 7.5
CUDA SETUP: Detected CUDA version 118
CUDA SETUP: Loading binary /usr/local/lib/python3.10/dist-packages/bitsandbytes/libbitsandbytes_cuda118.so...
WARNING:root:BetterTransformers probably doesn't work with PEFT adapters
WARNING:root:You should probably set bfloat16 or float16 to true to load the model in float16 for BetterTransformers
INFO:root:loading tokenizer... EleutherAI/pythia-1b-deduped
Using pad_token, but it is not set yet.
INFO:root:Loading prepared dataset from disk at last_run_prepared/38e3e506b1368f55cde052aaffa67e9a...
INFO:root:Prepared dataset loaded from disk...
INFO:root:loading model and peft_config...
WARNING:root:increasing model.config.max_position_embeddings to 2048
INFO:root:converting PEFT model w/ prepare_model_for_kbit_training
INFO:root:found linear modules: ['query_key_value', 'dense_h_to_4h', 'dense_4h_to_h', 'dense']
trainable params: 33,554,432 || all params: 642,682,880 || trainable%: 5.220993594850388
╭───────────────────── Traceback (most recent call last) ──────────────────────╮
│ /content/axolotl/scripts/finetune.py:352 in <module> │
│ │
│ 349 │
│ 350 │
│ 351 if __name__ == "__main__": │
│ ❱ 352 │ fire.Fire(train) │
│ 353 │
│ │
│ /usr/local/lib/python3.10/dist-packages/fire/core.py:141 in Fire │
│ │
│ 138 │ context.update(caller_globals) │
│ 139 │ context.update(caller_locals) │
│ 140 │
│ ❱ 141 component_trace = _Fire(component, args, parsed_flag_args, context, │
│ 142 │
│ 143 if component_trace.HasError(): │
│ 144 │ _DisplayError(component_trace) │
│ │
│ /usr/local/lib/python3.10/dist-packages/fire/core.py:475 in _Fire │
│ │
│ 472 │ is_class = inspect.isclass(component) │
│ 473 │ │
│ 474 │ try: │
│ ❱ 475 │ │ component, remaining_args = _CallAndUpdateTrace( │
│ 476 │ │ │ component, │
│ 477 │ │ │ remaining_args, │
│ 478 │ │ │ component_trace, │
│ │
│ /usr/local/lib/python3.10/dist-packages/fire/core.py:691 in │
│ _CallAndUpdateTrace │
│ │
│ 688 │ loop = asyncio.get_event_loop() │
│ 689 │ component = loop.run_until_complete(fn(*varargs, **kwargs)) │
│ 690 else: │
│ ❱ 691 │ component = fn(*varargs, **kwargs) │
│ 692 │
│ 693 if treatment == 'class': │
│ 694 │ action = trace.INSTANTIATED_CLASS │
│ │
│ /content/axolotl/scripts/finetune.py:251 in train │
│ │
│ 248 │ │
│ 249 │ # Load the model and tokenizer │
│ 250 │ logging.info("loading model and peft_config...") │
│ ❱ 251 │ model, peft_config = load_model( │
│ 252 │ │ cfg.base_model, │
│ 253 │ │ cfg.base_model_config, │
│ 254 │ │ cfg.model_type, │
│ │
│ /content/axolotl/src/axolotl/utils/models.py:354 in load_model │
│ │
│ 351 │ model.config.use_cache = False │
│ 352 │ │
│ 353 │ if cfg.flash_optimum: │
│ ❱ 354 │ │ model = BetterTransformer.transform(model) │
│ 355 │ │
│ 356 │ # TODO resume_from_checkpoint handling │
│ 357 │ return model, lora_config │
│ │
│ /usr/lib/python3.10/contextlib.py:79 in inner │
│ │
│ 76 │ │ @wraps(func) │
│ 77 │ │ def inner(*args, **kwds): │
│ 78 │ │ │ with self._recreate_cm(): │
│ ❱ 79 │ │ │ │ return func(*args, **kwds) │
│ 80 │ │ return inner │
│ 81 │
│ 82 │
│ │
│ /usr/local/lib/python3.10/dist-packages/optimum/bettertransformer/transforma │
│ tion.py:229 in transform │
│ │
│ 226 │ │ │
│ 227 │ │ if load_accelerate: │
│ 228 │ │ │ # Remove the hooks from the original model to avoid weight │
│ ❱ 229 │ │ │ remove_hook_from_module(model, recurse=True) │
│ 230 │ │ │
│ 231 │ │ if keep_original_model: │
│ 232 │ │ │ try: │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/hooks.py:187 in │
│ remove_hook_from_module │
│ │
│ 184 │ │
│ 185 │ if hasattr(module, "_hf_hook"): │
│ 186 │ │ module._hf_hook.detach_hook(module) │
│ ❱ 187 │ │ delattr(module, "_hf_hook") │
│ 188 │ │
│ 189 │ if hasattr(module, "_old_forward"): │
│ 190 │ │ module.forward = module._old_forward │
│ │
│ /usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py:1685 in │
│ __delattr__ │
│ │
│ 1682 │ │ elif name in self._modules: │
│ 1683 │ │ │ del self._modules[name] │
│ 1684 │ │ else: │
│ ❱ 1685 │ │ │ super().__delattr__(name) │
│ 1686 │ │
│ 1687 │ def _register_state_dict_hook(self, hook): │
│ 1688 │ │ r"""These hooks will be called with arguments: `self`, `state │
╰──────────────────────────────────────────────────────────────────────────────╯
AttributeError: _hf_hook
╭───────────────────── Traceback (most recent call last) ──────────────────────╮
│ /usr/local/bin/accelerate:8 in <module> │
│ │
│ 5 from accelerate.commands.accelerate_cli import main │
│ 6 if __name__ == '__main__': │
│ 7 │ sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) │
│ ❱ 8 │ sys.exit(main()) │
│ 9 │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/commands/accelerate_cli.p │
│ y:45 in main │
│ │
│ 42 │ │ exit(1) │
│ 43 │ │
│ 44 │ # Run │
│ ❱ 45 │ args.func(args) │
│ 46 │
│ 47 │
│ 48 if __name__ == "__main__": │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/commands/launch.py:941 in │
│ launch_command │
│ │
│ 938 │ elif defaults is not None and defaults.compute_environment == Comp │
│ 939 │ │ sagemaker_launcher(defaults, args) │
│ 940 │ else: │
│ ❱ 941 │ │ simple_launcher(args) │
│ 942 │
│ 943 │
│ 944 def main(): │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/commands/launch.py:603 in │
│ simple_launcher │
│ │
│ 600 │ process.wait() │
│ 601 │ if process.returncode != 0: │
│ 602 │ │ if not args.quiet: │
│ ❱ 603 │ │ │ raise subprocess.CalledProcessError(returncode=process.ret │
│ 604 │ │ else: │
│ 605 │ │ │ sys.exit(1) │
│ 606 │
╰──────────────────────────────────────────────────────────────────────────────╯
CalledProcessError: Command '['/usr/bin/python3', 'scripts/finetune.py',
'pythia-1b-qlora.yml']' returned non-zero exit status 1.
✅ flash_optimum + ft + pythia¶
In [14]:
%env WANDB_MODE=online
env: WANDB_MODE=online
In [16]:
import wandb
wandb.login()
wandb: Appending key for api.wandb.ai to your netrc file: /root/.netrc
Out[16]:
True
In [17]:
%%writefile pythia-1b.yml
base_model: EleutherAI/pythia-1b-deduped
base_model_config: EleutherAI/pythia-1b-deduped
base_model_ignore_patterns: pytorch* # prefer safetensors
model_type: GPTNeoXForCausalLM
tokenizer_type: AutoTokenizer
load_in_8bit: false
load_in_4bit: false
gptq: false
device_map: auto
datasets:
- path: QingyiSi/Alpaca-CoT
data_files:
- Chain-of-Thought/formatted_cot_data/gsm8k_train.json
type: "alpaca:chat"
dataset_prepared_path: last_run_prepared
val_set_size: 0.05
adapter:
lora_model_dir:
sequence_len: 2048
max_packed_sequence_len: 2048
lora_r: 64
lora_alpha: 32
lora_dropout: 0.0
lora_target_modules:
lora_target_linear: true
lora_fan_in_fan_out: true # pythia/GPTNeoX lora specific
wandb_project: flash_optimum
wandb_watch:
wandb_run_id:
wandb_log_model:
output_dir: ./pythia-12b
gradient_accumulation_steps: 1
micro_batch_size: 1
num_epochs: 5
learning_rate: 0.00003
optimizer: adamw_bnb_8bit
lr_scheduler: cosine
train_on_inputs: false
group_by_length: false
bf16: false
fp16: false
tf32: false
flash_optimum: true
early_stopping_patience:
resume_from_checkpoint:
local_rank:
gradient_checkpointing: true
fsdp:
fsdp_config:
collator_pad_to_longest: true
Overwriting pythia-1b.yml
In [18]:
!python scripts/finetune.py pythia-1b.yml
2023-06-16 04:01:15.170906: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
===================================BUG REPORT===================================
Welcome to bitsandbytes. For bug reports, please run
python -m bitsandbytes
and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues
================================================================================
bin /usr/local/lib/python3.10/dist-packages/bitsandbytes/libbitsandbytes_cuda118.so
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: /usr/lib64-nvidia did not contain ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] as expected! Searching further paths...
warn(msg)
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/sys/fs/cgroup/memory.events /var/colab/cgroup/jupyter-children/memory.events')}
warn(msg)
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('//172.28.0.1'), PosixPath('http'), PosixPath('8013')}
warn(msg)
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('//colab.research.google.com/tun/m/cc48301118ce562b961b3c22d803539adc1e0c19/gpu-t4-s-19a78udzj535v --tunnel_background_save_delay=10s --tunnel_periodic_background_save_frequency=30m0s --enable_output_coalescing=true --output_coalescing_required=true'), PosixPath('--logtostderr --listen_host=172.28.0.12 --target_host=172.28.0.12 --tunnel_background_save_url=https')}
warn(msg)
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/env/python')}
warn(msg)
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('module'), PosixPath('//ipykernel.pylab.backend_inline')}
warn(msg)
CUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching in backup paths...
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: Found duplicate ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] files: {PosixPath('/usr/local/cuda/lib64/libcudart.so'), PosixPath('/usr/local/cuda/lib64/libcudart.so.11.0')}.. We'll flip a coin and try one of these, in order to fail forward.
Either way, this might cause trouble in the future:
If you get `CUDA error: invalid device function` errors, the above might be the cause and the solution is to make sure only one ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] in the paths that we search based on your env.
warn(msg)
CUDA SETUP: CUDA runtime path found: /usr/local/cuda/lib64/libcudart.so
CUDA SETUP: Highest compute capability among GPUs detected: 7.5
CUDA SETUP: Detected CUDA version 118
CUDA SETUP: Loading binary /usr/local/lib/python3.10/dist-packages/bitsandbytes/libbitsandbytes_cuda118.so...
WARNING:root:You should probably set bfloat16 or float16 to true to load the model in float16 for BetterTransformers
INFO:root:loading tokenizer... EleutherAI/pythia-1b-deduped
Using pad_token, but it is not set yet.
INFO:root:Loading prepared packed dataset from disk at last_run_prepared/779a46d5ee8ddf774b5f7e31c9fc2b97...
INFO:root:Prepared packed dataset loaded from disk...
INFO:root:loading model and peft_config...
WARNING:accelerate.utils.modeling:The model weights are not tied. Please use the `tie_weights` method before using the `infer_auto_device` function.
WARNING:root:increasing model.config.max_position_embeddings to 2048
WARNING:root:For training, the BetterTransformer implementation for gpt_neox architecture currently does not support padding as fused kernels do not support custom attention masks. Beware that passing padded batched training data may result in unexpected outputs.
INFO:root:Compiling torch model
INFO:root:Starting trainer...
wandb: Currently logged in as: utensil. Use `wandb login --relogin` to force relogin
wandb: Tracking run with wandb version 0.15.4
wandb: Run data is saved locally in /content/axolotl/wandb/run-20230616_040131-1bnglwzh
wandb: Run `wandb offline` to turn off syncing.
wandb: Syncing run eager-deluge-1
wandb: ⭐️ View project at https://wandb.ai/utensil/flash_optimum
wandb: 🚀 View run at https://wandb.ai/utensil/flash_optimum/runs/1bnglwzh
0% 0/3040 [00:00<?, ?it/s]You're using a GPTNeoXTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.
{'loss': 1.6218, 'learning_rate': 3.2967032967032968e-06, 'epoch': 0.02}
0% 10/3040 [00:50<4:22:10, 5.19s/it]
0% 0/32 [00:00<?, ?it/s]
6% 2/32 [00:01<00:17, 1.70it/s]
9% 3/32 [00:02<00:24, 1.20it/s]
12% 4/32 [00:03<00:28, 1.00s/it]
16% 5/32 [00:04<00:29, 1.11s/it]
19% 6/32 [00:06<00:29, 1.13s/it]
22% 7/32 [00:07<00:29, 1.19s/it]
25% 8/32 [00:08<00:29, 1.23s/it]
28% 9/32 [00:09<00:28, 1.22s/it]
31% 10/32 [00:11<00:26, 1.18s/it]
34% 11/32 [00:12<00:24, 1.15s/it]
38% 12/32 [00:13<00:23, 1.15s/it]
41% 13/32 [00:14<00:22, 1.20s/it]
44% 14/32 [00:15<00:22, 1.24s/it]
47% 15/32 [00:17<00:20, 1.21s/it]
50% 16/32 [00:18<00:19, 1.23s/it]
53% 17/32 [00:19<00:18, 1.24s/it]
56% 18/32 [00:20<00:17, 1.25s/it]
59% 19/32 [00:21<00:15, 1.21s/it]
62% 20/32 [00:23<00:14, 1.23s/it]
66% 21/32 [00:24<00:13, 1.22s/it]
69% 22/32 [00:25<00:11, 1.18s/it]
72% 23/32 [00:26<00:10, 1.21s/it]
75% 24/32 [00:28<00:09, 1.24s/it]
78% 25/32 [00:29<00:08, 1.21s/it]
81% 26/32 [00:30<00:07, 1.23s/it]
84% 27/32 [00:31<00:06, 1.26s/it]
88% 28/32 [00:32<00:04, 1.22s/it]
91% 29/32 [00:34<00:03, 1.24s/it]
94% 30/32 [00:35<00:02, 1.27s/it]
97% 31/32 [00:36<00:01, 1.29s/it]
{'eval_loss': 1.4932047128677368, 'eval_runtime': 39.1586, 'eval_samples_per_second': 0.817, 'eval_steps_per_second': 0.817, 'epoch': 0.02}
0% 10/3040 [01:29<4:22:10, 5.19s/it]
100% 32/32 [00:38<00:00, 1.19s/it]
{'loss': 1.4783, 'learning_rate': 6.5934065934065935e-06, 'epoch': 0.03}
1% 20/3040 [02:26<5:07:02, 6.10s/it]
0% 0/32 [00:00<?, ?it/s]
6% 2/32 [00:01<00:18, 1.60it/s]
9% 3/32 [00:02<00:25, 1.13it/s]
12% 4/32 [00:03<00:29, 1.06s/it]
16% 5/32 [00:05<00:31, 1.17s/it]
19% 6/32 [00:06<00:30, 1.19s/it]
22% 7/32 [00:07<00:31, 1.24s/it]
25% 8/32 [00:09<00:30, 1.28s/it]
28% 9/32 [00:10<00:29, 1.27s/it]
31% 10/32 [00:11<00:27, 1.23s/it]
34% 11/32 [00:12<00:25, 1.20s/it]
38% 12/32 [00:13<00:23, 1.19s/it]
41% 13/32 [00:15<00:23, 1.24s/it]
44% 14/32 [00:16<00:22, 1.28s/it]
47% 15/32 [00:17<00:21, 1.24s/it]
50% 16/32 [00:19<00:20, 1.27s/it]
53% 17/32 [00:20<00:18, 1.26s/it]
56% 18/32 [00:21<00:17, 1.28s/it]
59% 19/32 [00:22<00:16, 1.25s/it]
62% 20/32 [00:24<00:15, 1.27s/it]
66% 21/32 [00:25<00:13, 1.27s/it]
69% 22/32 [00:26<00:12, 1.22s/it]
72% 23/32 [00:27<00:11, 1.25s/it]
75% 24/32 [00:29<00:10, 1.29s/it]
78% 25/32 [00:30<00:08, 1.25s/it]
81% 26/32 [00:31<00:07, 1.27s/it]
84% 27/32 [00:33<00:06, 1.30s/it]
88% 28/32 [00:34<00:05, 1.26s/it]
91% 29/32 [00:35<00:03, 1.28s/it]
94% 30/32 [00:36<00:02, 1.31s/it]
97% 31/32 [00:38<00:01, 1.33s/it]
{'eval_loss': 1.7296018600463867, 'eval_runtime': 40.6072, 'eval_samples_per_second': 0.788, 'eval_steps_per_second': 0.788, 'epoch': 0.03}
1% 20/3040 [03:07<5:07:02, 6.10s/it]
100% 32/32 [00:39<00:00, 1.22s/it]
{'loss': 1.9778, 'learning_rate': 9.89010989010989e-06, 'epoch': 0.05}
1% 30/3040 [04:04<5:11:13, 6.20s/it]
0% 0/32 [00:00<?, ?it/s]
6% 2/32 [00:01<00:18, 1.60it/s]
9% 3/32 [00:02<00:25, 1.14it/s]
12% 4/32 [00:03<00:29, 1.05s/it]
16% 5/32 [00:05<00:31, 1.17s/it]
19% 6/32 [00:06<00:30, 1.19s/it]
22% 7/32 [00:07<00:31, 1.24s/it]
25% 8/32 [00:09<00:30, 1.28s/it]
28% 9/32 [00:10<00:29, 1.27s/it]
31% 10/32 [00:11<00:27, 1.23s/it]
34% 11/32 [00:12<00:25, 1.20s/it]
38% 12/32 [00:13<00:24, 1.20s/it]
41% 13/32 [00:15<00:23, 1.24s/it]
44% 14/32 [00:16<00:23, 1.28s/it]
47% 15/32 [00:17<00:21, 1.25s/it]
50% 16/32 [00:19<00:20, 1.28s/it]
53% 17/32 [00:20<00:18, 1.27s/it]
56% 18/32 [00:21<00:18, 1.29s/it]
59% 19/32 [00:22<00:16, 1.25s/it]
62% 20/32 [00:24<00:15, 1.28s/it]
66% 21/32 [00:25<00:13, 1.27s/it]
69% 22/32 [00:26<00:12, 1.23s/it]
72% 23/32 [00:27<00:11, 1.26s/it]
75% 24/32 [00:29<00:10, 1.30s/it]
78% 25/32 [00:30<00:08, 1.25s/it]
81% 26/32 [00:31<00:07, 1.28s/it]
84% 27/32 [00:33<00:06, 1.31s/it]
88% 28/32 [00:34<00:05, 1.27s/it]
91% 29/32 [00:35<00:03, 1.28s/it]
94% 30/32 [00:37<00:02, 1.31s/it]
97% 31/32 [00:38<00:01, 1.33s/it]
{'eval_loss': 2.3409981727600098, 'eval_runtime': 40.6999, 'eval_samples_per_second': 0.786, 'eval_steps_per_second': 0.786, 'epoch': 0.05}
1% 30/3040 [04:45<5:11:13, 6.20s/it]
100% 32/32 [00:39<00:00, 1.22s/it]
{'loss': 2.2382, 'learning_rate': 1.3186813186813187e-05, 'epoch': 0.07}
1% 40/3040 [05:42<5:12:55, 6.26s/it]
0% 0/32 [00:00<?, ?it/s]
6% 2/32 [00:01<00:18, 1.59it/s]
9% 3/32 [00:02<00:25, 1.13it/s]
12% 4/32 [00:03<00:29, 1.06s/it]
16% 5/32 [00:05<00:31, 1.18s/it]
19% 6/32 [00:06<00:31, 1.20s/it]
22% 7/32 [00:07<00:31, 1.25s/it]
25% 8/32 [00:09<00:31, 1.29s/it]
28% 9/32 [00:10<00:29, 1.28s/it]
31% 10/32 [00:11<00:27, 1.24s/it]
34% 11/32 [00:12<00:25, 1.21s/it]
38% 12/32 [00:13<00:24, 1.21s/it]
41% 13/32 [00:15<00:23, 1.25s/it]
44% 14/32 [00:16<00:23, 1.29s/it]
47% 15/32 [00:17<00:21, 1.26s/it]
50% 16/32 [00:19<00:20, 1.29s/it]
53% 17/32 [00:20<00:19, 1.28s/it]
56% 18/32 [00:21<00:18, 1.30s/it]
59% 19/32 [00:23<00:16, 1.27s/it]
62% 20/32 [00:24<00:15, 1.29s/it]
66% 21/32 [00:25<00:14, 1.28s/it]
69% 22/32 [00:26<00:12, 1.24s/it]
72% 23/32 [00:28<00:11, 1.27s/it]
75% 24/32 [00:29<00:10, 1.31s/it]
78% 25/32 [00:30<00:08, 1.27s/it]
81% 26/32 [00:32<00:07, 1.29s/it]
84% 27/32 [00:33<00:06, 1.32s/it]
88% 28/32 [00:34<00:05, 1.28s/it]
91% 29/32 [00:35<00:03, 1.30s/it]
94% 30/32 [00:37<00:02, 1.32s/it]
97% 31/32 [00:38<00:01, 1.34s/it]
{'eval_loss': 2.742034435272217, 'eval_runtime': 41.0436, 'eval_samples_per_second': 0.78, 'eval_steps_per_second': 0.78, 'epoch': 0.07}
1% 40/3040 [06:23<5:12:55, 6.26s/it]
100% 32/32 [00:39<00:00, 1.23s/it]
{'loss': 3.349, 'learning_rate': 1.6483516483516486e-05, 'epoch': 0.08}
2% 50/3040 [07:19<5:00:15, 6.03s/it]
0% 0/32 [00:00<?, ?it/s]
6% 2/32 [00:01<00:18, 1.61it/s]
9% 3/32 [00:02<00:25, 1.14it/s]
12% 4/32 [00:03<00:29, 1.05s/it]
16% 5/32 [00:05<00:31, 1.17s/it]
19% 6/32 [00:06<00:31, 1.19s/it]
22% 7/32 [00:07<00:31, 1.24s/it]
25% 8/32 [00:09<00:30, 1.29s/it]
28% 9/32 [00:10<00:29, 1.28s/it]
31% 10/32 [00:11<00:27, 1.24s/it]
34% 11/32 [00:12<00:25, 1.20s/it]
38% 12/32 [00:13<00:24, 1.20s/it]
41% 13/32 [00:15<00:23, 1.25s/it]
44% 14/32 [00:16<00:23, 1.29s/it]
47% 15/32 [00:17<00:21, 1.25s/it]
50% 16/32 [00:19<00:20, 1.28s/it]
53% 17/32 [00:20<00:19, 1.27s/it]
56% 18/32 [00:21<00:18, 1.30s/it]
59% 19/32 [00:22<00:16, 1.26s/it]
62% 20/32 [00:24<00:15, 1.29s/it]
66% 21/32 [00:25<00:14, 1.28s/it]
69% 22/32 [00:26<00:12, 1.24s/it]
72% 23/32 [00:28<00:11, 1.27s/it]
75% 24/32 [00:29<00:10, 1.30s/it]
78% 25/32 [00:30<00:08, 1.26s/it]
81% 26/32 [00:31<00:07, 1.28s/it]
84% 27/32 [00:33<00:06, 1.31s/it]
88% 28/32 [00:34<00:05, 1.27s/it]
91% 29/32 [00:35<00:03, 1.29s/it]
94% 30/32 [00:37<00:02, 1.32s/it]
97% 31/32 [00:38<00:01, 1.34s/it]
{'eval_loss': 3.4204564094543457, 'eval_runtime': 40.8771, 'eval_samples_per_second': 0.783, 'eval_steps_per_second': 0.783, 'epoch': 0.08}
2% 50/3040 [08:00<5:00:15, 6.03s/it]
100% 32/32 [00:39<00:00, 1.23s/it]
{'loss': 3.6366, 'learning_rate': 1.978021978021978e-05, 'epoch': 0.1}
2% 60/3040 [08:57<5:12:06, 6.28s/it]
0% 0/32 [00:00<?, ?it/s]
6% 2/32 [00:01<00:18, 1.60it/s]
9% 3/32 [00:02<00:25, 1.14it/s]
12% 4/32 [00:03<00:29, 1.05s/it]
16% 5/32 [00:05<00:31, 1.16s/it]
19% 6/32 [00:06<00:30, 1.19s/it]
22% 7/32 [00:07<00:31, 1.24s/it]
25% 8/32 [00:09<00:30, 1.28s/it]
28% 9/32 [00:10<00:29, 1.27s/it]
31% 10/32 [00:11<00:27, 1.23s/it]
34% 11/32 [00:12<00:25, 1.20s/it]
38% 12/32 [00:13<00:23, 1.20s/it]
41% 13/32 [00:15<00:23, 1.25s/it]
44% 14/32 [00:16<00:23, 1.29s/it]
47% 15/32 [00:17<00:21, 1.25s/it]
50% 16/32 [00:19<00:20, 1.28s/it]
53% 17/32 [00:20<00:19, 1.27s/it]
56% 18/32 [00:21<00:18, 1.30s/it]
59% 19/32 [00:22<00:16, 1.26s/it]
62% 20/32 [00:24<00:15, 1.29s/it]
66% 21/32 [00:25<00:14, 1.28s/it]
69% 22/32 [00:26<00:12, 1.24s/it]
72% 23/32 [00:27<00:11, 1.27s/it]
75% 24/32 [00:29<00:10, 1.30s/it]
78% 25/32 [00:30<00:08, 1.26s/it]
81% 26/32 [00:31<00:07, 1.28s/it]
84% 27/32 [00:33<00:06, 1.31s/it]
88% 28/32 [00:34<00:05, 1.27s/it]
91% 29/32 [00:35<00:03, 1.29s/it]
94% 30/32 [00:37<00:02, 1.32s/it]
97% 31/32 [00:38<00:01, 1.34s/it]
{'eval_loss': 4.200768947601318, 'eval_runtime': 40.8309, 'eval_samples_per_second': 0.784, 'eval_steps_per_second': 0.784, 'epoch': 0.1}
2% 60/3040 [09:38<5:12:06, 6.28s/it]
100% 32/32 [00:39<00:00, 1.23s/it]
{'loss': 4.4211, 'learning_rate': 2.307692307692308e-05, 'epoch': 0.12}
2% 70/3040 [10:35<5:09:27, 6.25s/it]
0% 0/32 [00:00<?, ?it/s]
6% 2/32 [00:01<00:18, 1.61it/s]
9% 3/32 [00:02<00:25, 1.14it/s]
12% 4/32 [00:03<00:29, 1.05s/it]
16% 5/32 [00:05<00:31, 1.17s/it]
19% 6/32 [00:06<00:30, 1.19s/it]
22% 7/32 [00:07<00:31, 1.24s/it]
25% 8/32 [00:09<00:30, 1.28s/it]
28% 9/32 [00:10<00:29, 1.27s/it]
31% 10/32 [00:11<00:27, 1.23s/it]
34% 11/32 [00:12<00:25, 1.20s/it]
38% 12/32 [00:13<00:23, 1.20s/it]
41% 13/32 [00:15<00:23, 1.24s/it]
44% 14/32 [00:16<00:23, 1.28s/it]
47% 15/32 [00:17<00:21, 1.25s/it]
50% 16/32 [00:19<00:20, 1.28s/it]
53% 17/32 [00:20<00:19, 1.27s/it]
56% 18/32 [00:21<00:18, 1.29s/it]
59% 19/32 [00:22<00:16, 1.26s/it]╭───────────────────── Traceback (most recent call last) ──────────────────────╮
│ /content/axolotl/scripts/finetune.py:352 in <module> │
│ │
│ 349 │
│ 350 │
│ 351 if __name__ == "__main__": │
│ ❱ 352 │ fire.Fire(train) │
│ 353 │
│ │
│ /usr/local/lib/python3.10/dist-packages/fire/core.py:141 in Fire │
│ │
│ 138 │ context.update(caller_globals) │
│ 139 │ context.update(caller_locals) │
│ 140 │
│ ❱ 141 component_trace = _Fire(component, args, parsed_flag_args, context, │
│ 142 │
│ 143 if component_trace.HasError(): │
│ 144 │ _DisplayError(component_trace) │
│ │
│ /usr/local/lib/python3.10/dist-packages/fire/core.py:475 in _Fire │
│ │
│ 472 │ is_class = inspect.isclass(component) │
│ 473 │ │
│ 474 │ try: │
│ ❱ 475 │ │ component, remaining_args = _CallAndUpdateTrace( │
│ 476 │ │ │ component, │
│ 477 │ │ │ remaining_args, │
│ 478 │ │ │ component_trace, │
│ │
│ /usr/local/lib/python3.10/dist-packages/fire/core.py:691 in │
│ _CallAndUpdateTrace │
│ │
│ 688 │ loop = asyncio.get_event_loop() │
│ 689 │ component = loop.run_until_complete(fn(*varargs, **kwargs)) │
│ 690 else: │
│ ❱ 691 │ component = fn(*varargs, **kwargs) │
│ 692 │
│ 693 if treatment == 'class': │
│ 694 │ action = trace.INSTANTIATED_CLASS │
│ │
│ /content/axolotl/scripts/finetune.py:335 in train │
│ │
│ 332 │ │ with torch.backends.cuda.sdp_kernel( │
│ 333 │ │ │ enable_flash=True, enable_math=True, enable_mem_efficient= │
│ 334 │ │ ): │
│ ❱ 335 │ │ │ trainer.train(resume_from_checkpoint=resume_from_checkpoin │
│ 336 │ else: │
│ 337 │ │ trainer.train(resume_from_checkpoint=resume_from_checkpoint) │
│ 338 │
│ │
│ /usr/local/lib/python3.10/dist-packages/transformers/trainer.py:1540 in │
│ train │
│ │
│ 1537 │ │ inner_training_loop = find_executable_batch_size( │
│ 1538 │ │ │ self._inner_training_loop, self._train_batch_size, args.a │
│ 1539 │ │ ) │
│ ❱ 1540 │ │ return inner_training_loop( │
│ 1541 │ │ │ args=args, │
│ 1542 │ │ │ resume_from_checkpoint=resume_from_checkpoint, │
│ 1543 │ │ │ trial=trial, │
│ │
│ /usr/local/lib/python3.10/dist-packages/transformers/trainer.py:1884 in │
│ _inner_training_loop │
│ │
│ 1881 │ │ │ │ │ self.state.epoch = epoch + (step + 1 + steps_skip │
│ 1882 │ │ │ │ │ self.control = self.callback_handler.on_step_end( │
│ 1883 │ │ │ │ │ │
│ ❱ 1884 │ │ │ │ │ self._maybe_log_save_evaluate(tr_loss, model, tri │
│ 1885 │ │ │ │ else: │
│ 1886 │ │ │ │ │ self.control = self.callback_handler.on_substep_e │
│ 1887 │
│ │
│ /usr/local/lib/python3.10/dist-packages/transformers/trainer.py:2185 in │
│ _maybe_log_save_evaluate │
│ │
│ 2182 │ │ │ │ │ ) │
│ 2183 │ │ │ │ │ metrics.update(dataset_metrics) │
│ 2184 │ │ │ else: │
│ ❱ 2185 │ │ │ │ metrics = self.evaluate(ignore_keys=ignore_keys_for_e │
│ 2186 │ │ │ self._report_to_hp_search(trial, self.state.global_step, │
│ 2187 │ │ │ │
│ 2188 │ │ │ # Run delayed LR scheduler now that metrics are populated │
│ │
│ /usr/local/lib/python3.10/dist-packages/transformers/trainer.py:2931 in │
│ evaluate │
│ │
│ 2928 │ │ start_time = time.time() │
│ 2929 │ │ │
│ 2930 │ │ eval_loop = self.prediction_loop if self.args.use_legacy_pred │
│ ❱ 2931 │ │ output = eval_loop( │
│ 2932 │ │ │ eval_dataloader, │
│ 2933 │ │ │ description="Evaluation", │
│ 2934 │ │ │ # No point gathering the predictions if there are no metr │
│ │
│ /usr/local/lib/python3.10/dist-packages/transformers/trainer.py:3120 in │
│ evaluation_loop │
│ │
│ 3117 │ │ │ │ │ batch_size = observed_batch_size │
│ 3118 │ │ │ │
│ 3119 │ │ │ # Prediction step │
│ ❱ 3120 │ │ │ loss, logits, labels = self.prediction_step(model, inputs │
│ 3121 │ │ │ inputs_decode = self._prepare_input(inputs["input_ids"]) │
│ 3122 │ │ │ │
│ 3123 │ │ │ if is_torch_tpu_available(): │
│ │
│ /usr/local/lib/python3.10/dist-packages/transformers/trainer.py:3363 in │
│ prediction_step │
│ │
│ 3360 │ │ │ else: │
│ 3361 │ │ │ │ if has_labels or loss_without_labels: │
│ 3362 │ │ │ │ │ with self.compute_loss_context_manager(): │
│ ❱ 3363 │ │ │ │ │ │ loss, outputs = self.compute_loss(model, inpu │
│ 3364 │ │ │ │ │ loss = loss.mean().detach() │
│ 3365 │ │ │ │ │ │
│ 3366 │ │ │ │ │ if isinstance(outputs, dict): │
│ │
│ /usr/local/lib/python3.10/dist-packages/transformers/trainer.py:2662 in │
│ compute_loss │
│ │
│ 2659 │ │ │ labels = inputs.pop("labels") │
│ 2660 │ │ else: │
│ 2661 │ │ │ labels = None │
│ ❱ 2662 │ │ outputs = model(**inputs) │
│ 2663 │ │ # Save past state if it exists │
│ 2664 │ │ # TODO: this needs to be fixed and made cleaner later. │
│ 2665 │ │ if self.args.past_index >= 0: │
│ │
│ /usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py:1501 in │
│ _call_impl │
│ │
│ 1498 │ │ if not (self._backward_hooks or self._backward_pre_hooks or s │
│ 1499 │ │ │ │ or _global_backward_pre_hooks or _global_backward_hoo │
│ 1500 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks │
│ ❱ 1501 │ │ │ return forward_call(*args, **kwargs) │
│ 1502 │ │ # Do not call functions when jit is used │
│ 1503 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │
│ 1504 │ │ backward_pre_hooks = [] │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/hooks.py:165 in │
│ new_forward │
│ │
│ 162 │ │ │ with torch.no_grad(): │
│ 163 │ │ │ │ output = old_forward(*args, **kwargs) │
│ 164 │ │ else: │
│ ❱ 165 │ │ │ output = old_forward(*args, **kwargs) │
│ 166 │ │ return module._hf_hook.post_forward(module, output) │
│ 167 │ │
│ 168 │ module.forward = new_forward │
│ │
│ /usr/local/lib/python3.10/dist-packages/transformers/models/gpt_neox/modelin │
│ g_gpt_neox.py:674 in forward │
│ │
│ 671 │ │ ```""" │
│ 672 │ │ return_dict = return_dict if return_dict is not None else sel │
│ 673 │ │ │
│ ❱ 674 │ │ outputs = self.gpt_neox( │
│ 675 │ │ │ input_ids, │
│ 676 │ │ │ attention_mask=attention_mask, │
│ 677 │ │ │ position_ids=position_ids, │
│ │
│ /usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py:1501 in │
│ _call_impl │
│ │
│ 1498 │ │ if not (self._backward_hooks or self._backward_pre_hooks or s │
│ 1499 │ │ │ │ or _global_backward_pre_hooks or _global_backward_hoo │
│ 1500 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks │
│ ❱ 1501 │ │ │ return forward_call(*args, **kwargs) │
│ 1502 │ │ # Do not call functions when jit is used │
│ 1503 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │
│ 1504 │ │ backward_pre_hooks = [] │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/hooks.py:165 in │
│ new_forward │
│ │
│ 162 │ │ │ with torch.no_grad(): │
│ 163 │ │ │ │ output = old_forward(*args, **kwargs) │
│ 164 │ │ else: │
│ ❱ 165 │ │ │ output = old_forward(*args, **kwargs) │
│ 166 │ │ return module._hf_hook.post_forward(module, output) │
│ 167 │ │
│ 168 │ module.forward = new_forward │
│ │
│ /usr/local/lib/python3.10/dist-packages/transformers/models/gpt_neox/modelin │
│ g_gpt_neox.py:564 in forward │
│ │
│ 561 │ │ │ │ │ head_mask[i], │
│ 562 │ │ │ │ ) │
│ 563 │ │ │ else: │
│ ❱ 564 │ │ │ │ outputs = layer( │
│ 565 │ │ │ │ │ hidden_states, │
│ 566 │ │ │ │ │ attention_mask=attention_mask, │
│ 567 │ │ │ │ │ position_ids=position_ids, │
│ │
│ /usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py:1501 in │
│ _call_impl │
│ │
│ 1498 │ │ if not (self._backward_hooks or self._backward_pre_hooks or s │
│ 1499 │ │ │ │ or _global_backward_pre_hooks or _global_backward_hoo │
│ 1500 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks │
│ ❱ 1501 │ │ │ return forward_call(*args, **kwargs) │
│ 1502 │ │ # Do not call functions when jit is used │
│ 1503 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │
│ 1504 │ │ backward_pre_hooks = [] │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/hooks.py:165 in │
│ new_forward │
│ │
│ 162 │ │ │ with torch.no_grad(): │
│ 163 │ │ │ │ output = old_forward(*args, **kwargs) │
│ 164 │ │ else: │
│ ❱ 165 │ │ │ output = old_forward(*args, **kwargs) │
│ 166 │ │ return module._hf_hook.post_forward(module, output) │
│ 167 │ │
│ 168 │ module.forward = new_forward │
│ │
│ /usr/local/lib/python3.10/dist-packages/transformers/models/gpt_neox/modelin │
│ g_gpt_neox.py:331 in forward │
│ │
│ 328 │ │ layer_past: Optional[Tuple[torch.Tensor]] = None, │
│ 329 │ │ output_attentions: Optional[bool] = False, │
│ 330 │ ): │
│ ❱ 331 │ │ attention_layer_outputs = self.attention( │
│ 332 │ │ │ self.input_layernorm(hidden_states), │
│ 333 │ │ │ attention_mask=attention_mask, │
│ 334 │ │ │ position_ids=position_ids, │
│ │
│ /usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py:1501 in │
│ _call_impl │
│ │
│ 1498 │ │ if not (self._backward_hooks or self._backward_pre_hooks or s │
│ 1499 │ │ │ │ or _global_backward_pre_hooks or _global_backward_hoo │
│ 1500 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks │
│ ❱ 1501 │ │ │ return forward_call(*args, **kwargs) │
│ 1502 │ │ # Do not call functions when jit is used │
│ 1503 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │
│ 1504 │ │ backward_pre_hooks = [] │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/hooks.py:165 in │
│ new_forward │
│ │
│ 162 │ │ │ with torch.no_grad(): │
│ 163 │ │ │ │ output = old_forward(*args, **kwargs) │
│ 164 │ │ else: │
│ ❱ 165 │ │ │ output = old_forward(*args, **kwargs) │
│ 166 │ │ return module._hf_hook.post_forward(module, output) │
│ 167 │ │
│ 168 │ module.forward = new_forward │
│ │
│ /usr/local/lib/python3.10/dist-packages/optimum/bettertransformer/models/dec │
│ oder_models.py:135 in forward │
│ │
│ 132 │ │
│ 133 │ def forward(self, *args, **kwargs): │
│ 134 │ │ super().forward_checker() │
│ ❱ 135 │ │ return super().forward(*args, **kwargs) │
│ 136 │
│ 137 │
│ 138 class GPTNeoAttentionLayerBetterTransformer(BetterTransformerBaseLayer │
│ │
│ /usr/local/lib/python3.10/dist-packages/transformers/models/gpt_neox/modelin │
│ g_gpt_neox.py:149 in forward │
│ │
│ 146 │ │ seq_len = key.shape[-2] │
│ 147 │ │ if has_layer_past: │
│ 148 │ │ │ seq_len += layer_past[0].shape[-2] │
│ ❱ 149 │ │ cos, sin = self.rotary_emb(value, seq_len=seq_len) │
│ 150 │ │ query, key = apply_rotary_pos_emb(query_rot, key_rot, cos, si │
│ 151 │ │ query = torch.cat((query, query_pass), dim=-1) │
│ 152 │ │ key = torch.cat((key, key_pass), dim=-1) │
│ │
│ /usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py:1501 in │
│ _call_impl │
│ │
│ 1498 │ │ if not (self._backward_hooks or self._backward_pre_hooks or s │
│ 1499 │ │ │ │ or _global_backward_pre_hooks or _global_backward_hoo │
│ 1500 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks │
│ ❱ 1501 │ │ │ return forward_call(*args, **kwargs) │
│ 1502 │ │ # Do not call functions when jit is used │
│ 1503 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │
│ 1504 │ │ backward_pre_hooks = [] │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/hooks.py:165 in │
│ new_forward │
│ │
│ 162 │ │ │ with torch.no_grad(): │
│ 163 │ │ │ │ output = old_forward(*args, **kwargs) │
│ 164 │ │ else: │
│ ❱ 165 │ │ │ output = old_forward(*args, **kwargs) │
│ 166 │ │ return module._hf_hook.post_forward(module, output) │
│ 167 │ │
│ 168 │ module.forward = new_forward │
│ │
│ /usr/local/lib/python3.10/dist-packages/transformers/models/gpt_neox/modelin │
│ g_gpt_neox.py:278 in forward │
│ │
│ 275 │ │ │ emb = torch.cat((freqs, freqs), dim=-1).to(x.device) │
│ 276 │ │ │ self.cos_cached = emb.cos()[None, None, :, :] │
│ 277 │ │ │ self.sin_cached = emb.sin()[None, None, :, :] │
│ ❱ 278 │ │ return self.cos_cached[:seq_len, ...].to(x.device), self.sin_ │
│ 279 │
│ 280 │
│ 281 def rotate_half(x): │
│ │
│ /content/axolotl/scripts/finetune.py:308 in <lambda> │
│ │
│ 305 │ │ │ sys.exit(0) │
│ 306 │ │ │
│ 307 │ │ signal.signal( │
│ ❱ 308 │ │ │ signal.SIGINT, lambda signum, frame: terminate_handler(sig │
│ 309 │ │ ) │
│ 310 │ │
│ 311 │ logging.info("Starting trainer...") │
│ │
│ /content/axolotl/scripts/finetune.py:303 in terminate_handler │
│ │
│ 300 │ │ │
│ 301 │ │ def terminate_handler(_, __, model): │
│ 302 │ │ │ if cfg.flash_optimum: │
│ ❱ 303 │ │ │ │ model = BetterTransformer.reverse(model) │
│ 304 │ │ │ model.save_pretrained(cfg.output_dir) │
│ 305 │ │ │ sys.exit(0) │
│ 306 │
│ │
│ /usr/local/lib/python3.10/dist-packages/optimum/bettertransformer/transforma │
│ tion.py:320 in reverse │
│ │
│ 317 │ │ │
│ 318 │ │ if config.model_type not in ["wav2vec2", "hubert"]: │
│ 319 │ │ │ with torch.device("meta"): │
│ ❱ 320 │ │ │ │ reversed_model = bt_model.__class__(config) │
│ 321 │ │ else: │
│ 322 │ │ │ # TODO: fix once this is fixed in pytorch │
│ 323 │ │ │ # reference: https://github.com/pytorch/pytorch/issues/964 │
╰──────────────────────────────────────────────────────────────────────────────╯
TypeError: OptimizedModule.__init__() missing 1 required positional argument:
'dynamo_ctx'
2% 70/3040 [11:02<7:48:43, 9.47s/it]
In [ ]: