Open In Colab

In [ ]:
#@title Keep this tab alive to prevent Colab from disconnecting you { display-mode: "form" }

#@markdown Press play on the music player that will appear below:
%%html
<audio src="https://oobabooga.github.io/silence.m4a" controls>
In [ ]:
!pip install nvitop

Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/
Collecting nvitop
  Downloading nvitop-1.1.2-py3-none-any.whl (206 kB)
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 206.8/206.8 kB 4.8 MB/s eta 0:00:00
Collecting nvidia-ml-py<11.526.0a0,>=11.450.51 (from nvitop)
  Downloading nvidia_ml_py-11.525.112-py3-none-any.whl (35 kB)
Requirement already satisfied: psutil>=5.6.6 in /usr/local/lib/python3.10/dist-packages (from nvitop) (5.9.5)
Requirement already satisfied: cachetools>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from nvitop) (5.3.0)
Requirement already satisfied: termcolor>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from nvitop) (2.3.0)
Installing collected packages: nvidia-ml-py, nvitop
Successfully installed nvidia-ml-py-11.525.112 nvitop-1.1.2
In [ ]:
# from nvitop import Device, ResourceMetricCollector, collect_in_background
# # import logging
# import os
# # logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO"))

# # logger = logging

# def on_collect(metrics):  # will be called periodically
#     print(metrics)
#     return True

# def on_stop(collector):  # will be called only once at stop
#     pass

# # Record metrics to the logger in the background every 5 seconds.
# # It will collect 5-second mean/min/max for each metric.
# collect_in_background(
#     on_collect,
#     ResourceMetricCollector(Device.cuda.all()),
#     interval=5.0,
#     on_stop=on_stop,
# )
Out[ ]:
<Thread(metrics-daemon, started daemon 140097992222464)>
In [ ]:
%cd /content/

/content
In [ ]:
%%writefile monit.py
from nvitop import ResourceMetricCollector, Device
import os
import time

collector = ResourceMetricCollector(devices=Device.cuda.all())

while True:
  with collector(tag='train'):
    metrics = collector.collect()
    print(metrics)
  time.sleep(5)

Writing monit.py
In [ ]:
%cd /content

/content
In [ ]:
%%writefile monit.py
from nvitop import Device, GpuProcess, NA, colored, host
import time

stat_begin = host.uptime()

while True:
  host.cpu_percent()
  #host.cpu_times_percent()
  host.net_io_counters().bytes_recv
  host.net_io_counters().bytes_sent
  host.disk_io_counters().read_bytes
  host.disk_io_counters().write_bytes
  host.disk_usage('/content').percent
  print(f'time: {host.uptime() - stat_begin}')
  host.memory_percent()

  print('cpu%', host.cpu_percent())
  print('mem%', host.memory_percent())

  for device in Device.cuda.all():
    with device.oneshot():
      device.gpu_utilization()
      device.memory_utilization()
      device.encoder_utilization()
      device.decoder_utilization()
      print(device.compute_mode())
      print(device.utilization_rates())

  time.sleep(5)

Overwriting monit.py
In [ ]:
!python monit.py

time: 0.0012826919555664062
cpu% 0.0
mem% 43.8
UtilizationRates(gpu=0, memory=0, encoder=0, decoder=0)
Traceback (most recent call last):
  File "/content/monit.py", line 27, in <module>
    time.sleep(5)
KeyboardInterrupt
^C
In [ ]:
from nvitop import ResourceMetricCollector, Device
import os
import time

collector = ResourceMetricCollector(devices=Device.cuda.all())

METRIC_NAMES = {
    'cpu%': 'cpu_percent (%)',
    'mem%': 'memory_percent (%)',
    'mem': 'memory_used (GiB)',
    'gpu%': 'gpu_utilization (%)',

}

def get_metric_name(metric_name):
  return METRIC_NAMES[metric]

def get_metric(metrics, metric_name, tag='train', device='host', stats_type='mean'):
  metric_full_name = f'{tag}/{device}/{METRIC_NAMES[metric_name]}/{stats_type}',
  return metrics[metric_full_name]

def get_gpu_name():
  return f'cuda:{cuda_index} (gpu:{nvml_index})'

metrics = None
with collector(tag='train'):
  metrics = collector.collect()

metrics
In [ ]:
from nvitop import ResourceMetricCollector, Device
import os
import time

collector = ResourceMetricCollector(devices=Device.cuda.all())

metrics = None
with collector(tag='train'):
  metrics = collector.collect()

metrics
Out[ ]:
{'train/host/cpu_percent (%)/mean': 21.400000000000002,
 'train/host/cpu_percent (%)/min': 21.4,
 'train/host/cpu_percent (%)/max': 21.4,
 'train/host/memory_percent (%)/mean': 46.0,
 'train/host/memory_percent (%)/min': 46.0,
 'train/host/memory_percent (%)/max': 46.0,
 'train/host/swap_percent (%)/mean': 0.0,
 'train/host/swap_percent (%)/min': 0.0,
 'train/host/swap_percent (%)/max': 0.0,
 'train/host/memory_used (GiB)/mean': 5.018032073974609,
 'train/host/memory_used (GiB)/min': 5.018032073974609,
 'train/host/memory_used (GiB)/max': 5.018032073974609,
 'train/host/load_average (%) (1 min)/mean': 0.63,
 'train/host/load_average (%) (1 min)/min': 0.63,
 'train/host/load_average (%) (1 min)/max': 0.63,
 'train/host/load_average (%) (5 min)/mean': 0.53,
 'train/host/load_average (%) (5 min)/min': 0.53,
 'train/host/load_average (%) (5 min)/max': 0.53,
 'train/host/load_average (%) (15 min)/mean': 0.54,
 'train/host/load_average (%) (15 min)/min': 0.54,
 'train/host/load_average (%) (15 min)/max': 0.54,
 'train/cuda:0 (gpu:0)/memory_used (MiB)/mean': 14571.0,
 'train/cuda:0 (gpu:0)/memory_used (MiB)/min': 14571.0,
 'train/cuda:0 (gpu:0)/memory_used (MiB)/max': 14571.0,
 'train/cuda:0 (gpu:0)/memory_free (MiB)/mean': 530.8125,
 'train/cuda:0 (gpu:0)/memory_free (MiB)/min': 530.8125,
 'train/cuda:0 (gpu:0)/memory_free (MiB)/max': 530.8125,
 'train/cuda:0 (gpu:0)/memory_total (MiB)/mean': 15360.0,
 'train/cuda:0 (gpu:0)/memory_total (MiB)/min': 15360.0,
 'train/cuda:0 (gpu:0)/memory_total (MiB)/max': 15360.0,
 'train/cuda:0 (gpu:0)/memory_percent (%)/mean': 94.9,
 'train/cuda:0 (gpu:0)/memory_percent (%)/min': 94.9,
 'train/cuda:0 (gpu:0)/memory_percent (%)/max': 94.9,
 'train/cuda:0 (gpu:0)/gpu_utilization (%)/mean': 0.0,
 'train/cuda:0 (gpu:0)/gpu_utilization (%)/min': 0.0,
 'train/cuda:0 (gpu:0)/gpu_utilization (%)/max': 0.0,
 'train/cuda:0 (gpu:0)/memory_utilization (%)/mean': 0.0,
 'train/cuda:0 (gpu:0)/memory_utilization (%)/min': 0.0,
 'train/cuda:0 (gpu:0)/memory_utilization (%)/max': 0.0,
 'train/cuda:0 (gpu:0)/fan_speed (%)/mean': nan,
 'train/cuda:0 (gpu:0)/fan_speed (%)/min': nan,
 'train/cuda:0 (gpu:0)/fan_speed (%)/max': nan,
 'train/cuda:0 (gpu:0)/temperature (C)/mean': 58.0,
 'train/cuda:0 (gpu:0)/temperature (C)/min': 58.0,
 'train/cuda:0 (gpu:0)/temperature (C)/max': 58.0,
 'train/cuda:0 (gpu:0)/power_usage (W)/mean': 30.285,
 'train/cuda:0 (gpu:0)/power_usage (W)/min': 30.285,
 'train/cuda:0 (gpu:0)/power_usage (W)/max': 30.285,
 'train/duration (s)': 0.011947019999752229,
 'train/timestamp': 1687179233.5738797}
In [ ]:
devices=Device.cuda.all()
devices
Out[ ]:
[CudaDevice(cuda_index=0, nvml_index=0, name="Tesla T4", total_memory=15360MiB)]
In [ ]:
!pip install wandb

Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/
Collecting wandb
  Downloading wandb-0.15.4-py3-none-any.whl (2.1 MB)
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 2.1/2.1 MB 26.0 MB/s eta 0:00:00
Requirement already satisfied: Click!=8.0.0,>=7.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (8.1.3)
Collecting GitPython!=3.1.29,>=1.0.0 (from wandb)
  Downloading GitPython-3.1.31-py3-none-any.whl (184 kB)
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 184.3/184.3 kB 22.8 MB/s eta 0:00:00
Requirement already satisfied: requests<3,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (2.27.1)
Requirement already satisfied: psutil>=5.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (5.9.5)
Collecting sentry-sdk>=1.0.0 (from wandb)
  Downloading sentry_sdk-1.25.1-py2.py3-none-any.whl (206 kB)
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 206.7/206.7 kB 23.0 MB/s eta 0:00:00
Collecting docker-pycreds>=0.4.0 (from wandb)
  Downloading docker_pycreds-0.4.0-py2.py3-none-any.whl (9.0 kB)
Requirement already satisfied: PyYAML in /usr/local/lib/python3.10/dist-packages (from wandb) (6.0)
Collecting pathtools (from wandb)
  Downloading pathtools-0.1.2.tar.gz (11 kB)
  Preparing metadata (setup.py) ... done
Collecting setproctitle (from wandb)
  Downloading setproctitle-1.3.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (30 kB)
Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from wandb) (67.7.2)
Requirement already satisfied: appdirs>=1.4.3 in /usr/local/lib/python3.10/dist-packages (from wandb) (1.4.4)
Requirement already satisfied: protobuf!=4.21.0,<5,>=3.19.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (3.20.3)
Requirement already satisfied: six>=1.4.0 in /usr/local/lib/python3.10/dist-packages (from docker-pycreds>=0.4.0->wandb) (1.16.0)
Collecting gitdb<5,>=4.0.1 (from GitPython!=3.1.29,>=1.0.0->wandb)
  Downloading gitdb-4.0.10-py3-none-any.whl (62 kB)
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 62.7/62.7 kB 8.4 MB/s eta 0:00:00
Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.0.0->wandb) (1.26.15)
Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.0.0->wandb) (2022.12.7)
Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.0.0->wandb) (2.0.12)
Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.0.0->wandb) (3.4)
Collecting smmap<6,>=3.0.1 (from gitdb<5,>=4.0.1->GitPython!=3.1.29,>=1.0.0->wandb)
  Downloading smmap-5.0.0-py3-none-any.whl (24 kB)
Building wheels for collected packages: pathtools
  Building wheel for pathtools (setup.py) ... done
  Created wheel for pathtools: filename=pathtools-0.1.2-py3-none-any.whl size=8791 sha256=5e3aa73b796744ad494123c59019c8295e06a282890aabc043780e7d02331d98
  Stored in directory: /root/.cache/pip/wheels/e7/f3/22/152153d6eb222ee7a56ff8617d80ee5207207a8c00a7aab794
Successfully built pathtools
Installing collected packages: pathtools, smmap, setproctitle, sentry-sdk, docker-pycreds, gitdb, GitPython, wandb
Successfully installed GitPython-3.1.31 docker-pycreds-0.4.0 gitdb-4.0.10 pathtools-0.1.2 sentry-sdk-1.25.1 setproctitle-1.3.2 smmap-5.0.0 wandb-0.15.4
In [ ]:
import wandb
wandb.login()

wandb: Appending key for api.wandb.ai to your netrc file: /root/.netrc
Out[ ]:
True
In [ ]:
api = wandb.Api()
In [ ]:
run = api.from_path("utensil/falcon-qlora/o4myz1ep")
In [ ]:
type(run)
Out[ ]:
wandb.apis.public.Run
In [117]:
run.id
Out[117]:
'o4myz1ep'
In [119]:
run.name
Out[119]:
'run-1: 1b 4*2'
In [ ]:
run.summary
Out[ ]:
{'train/loss': 1.054, 'train/epoch': 0, 'train/global_step': 15, 'eval/steps_per_second': 0.202, 'eval/samples_per_second': 1.615, '_step': 16, '_runtime': 1407.197679519653, 'eval/loss': 0.8626449704170227, '_timestamp': 1686901335.9796736, 'eval/runtime': 589.4539, 'train/learning_rate': 0.00019999999010671495}
In [ ]:
run.history()
Out[ ]:
eval/loss train/loss train/epoch eval/samples_per_second _step _runtime _timestamp eval/runtime train/global_step train/learning_rate eval/steps_per_second
0 NaN 1.4081 0 NaN 0 17.713272 1.686900e+09 NaN 1 0.00002 NaN
1 NaN 1.4267 0 NaN 1 32.767708 1.686900e+09 NaN 2 0.00004 NaN
2 NaN 1.4222 0 NaN 2 47.414186 1.686900e+09 NaN 3 0.00006 NaN
3 NaN 1.5681 0 NaN 3 62.376609 1.686900e+09 NaN 4 0.00008 NaN
4 NaN 1.4507 0 NaN 4 77.411169 1.686900e+09 NaN 5 0.00010 NaN
5 0.805485 NaN 0 1.615 5 666.768924 1.686901e+09 589.3545 5 NaN 0.202
6 NaN 1.3742 0 NaN 6 681.548237 1.686901e+09 NaN 6 0.00012 NaN
7 NaN 1.2896 0 NaN 7 696.571450 1.686901e+09 NaN 7 0.00014 NaN
8 NaN 1.2189 0 NaN 8 711.589614 1.686901e+09 NaN 8 0.00016 NaN
9 NaN 1.0086 0 NaN 9 726.432516 1.686901e+09 NaN 9 0.00018 NaN
10 NaN 1.0008 0 NaN 10 741.416042 1.686901e+09 NaN 10 0.00020 NaN
11 0.862645 NaN 0 1.615 11 1330.873102 1.686901e+09 589.4539 10 NaN 0.202
12 NaN 1.2255 0 NaN 12 1348.252322 1.686901e+09 NaN 11 0.00020 NaN
13 NaN 1.0210 0 NaN 13 1363.273500 1.686901e+09 NaN 12 0.00020 NaN
14 NaN 0.9589 0 NaN 14 1377.600613 1.686901e+09 NaN 13 0.00020 NaN
15 NaN 1.0220 0 NaN 15 1392.408256 1.686901e+09 NaN 14 0.00020 NaN
16 NaN 1.0540 0 NaN 16 1407.197680 1.686901e+09 NaN 15 0.00020 NaN
In [ ]:
run.history(stream='system')
Out[ ]:
system.network.sent system.cpu.60.cpu_percent system.gpu.0.powerPercent system.cpu.52.cpu_percent system.network.recv system.cpu.42.cpu_percent system.cpu.78.cpu_percent system.gpu.1.temp system.cpu.22.cpu_percent system.cpu.9.cpu_percent ... system.cpu.55.cpu_percent system.cpu.8.cpu_percent system.cpu.59.cpu_percent system.cpu.38.cpu_percent system.cpu.18.cpu_percent system.cpu.71.cpu_percent system.cpu.108.cpu_percent system.cpu.17.cpu_percent system.cpu.14.cpu_percent system.cpu.30.cpu_percent
0 51167.07 0.03 81.52 0.00 37191.13 0.03 0.00 47.87 0.00 0.07 ... 0.00 0.07 8.68 0.00 0.07 0.14 0.00 37.81 0.13 0.07
1 128637.20 0.27 99.51 0.20 282360.73 0.20 2.15 58.07 0.13 0.47 ... 0.13 0.37 13.00 0.50 0.20 0.00 0.00 0.13 0.53 0.23
2 186626.80 0.17 99.04 0.03 492891.07 0.00 2.27 61.00 0.00 0.37 ... 0.00 0.07 8.71 0.00 0.07 0.70 0.03 0.10 0.33 0.03
3 225563.33 5.47 97.92 1.03 517867.07 0.20 1.33 61.47 0.13 0.33 ... 0.17 0.07 8.01 0.23 0.10 0.03 0.00 0.07 0.30 0.07
4 262362.27 8.25 101.26 1.13 541972.27 1.07 0.03 61.00 0.10 0.33 ... 0.03 0.03 0.00 0.00 0.00 0.03 0.03 0.07 0.03 0.17
5 311581.67 0.20 99.47 0.33 756349.80 0.79 0.03 60.93 0.13 0.40 ... 0.17 1.27 0.10 0.23 0.30 0.10 0.00 0.13 0.23 1.80
6 371698.60 0.03 95.46 0.07 999058.33 0.03 0.07 61.60 0.03 0.17 ... 0.00 0.37 0.00 0.03 0.10 0.03 0.03 0.13 0.03 1.33
7 413537.67 0.10 98.75 0.30 1024654.87 0.13 0.03 61.47 0.20 0.10 ... 0.13 0.50 0.13 0.13 0.33 0.10 0.00 0.20 0.27 1.27
8 449143.07 0.03 90.89 0.00 1048738.40 0.00 0.03 61.53 0.13 0.00 ... 0.00 0.37 0.03 0.00 0.03 0.03 0.00 0.30 0.00 1.03
9 494019.00 0.13 93.68 0.10 1233715.93 0.07 0.07 61.20 0.30 0.53 ... 0.10 0.73 0.07 0.07 0.07 0.07 0.00 0.10 0.20 1.20
10 558041.07 0.07 96.57 0.03 1513347.13 0.03 0.00 61.20 0.07 0.47 ... 0.00 11.73 0.03 0.03 0.07 0.70 0.00 0.07 0.13 1.23
11 627845.93 0.17 98.09 0.20 1556292.53 0.33 0.03 60.40 0.17 0.37 ... 0.10 21.95 0.17 0.10 0.07 1.36 0.00 0.13 0.43 1.23
12 748179.47 0.03 98.71 0.13 1605694.60 0.03 0.00 60.93 0.10 0.13 ... 0.03 0.27 0.07 0.00 0.00 0.93 0.00 0.17 0.60 1.10
13 810007.27 0.17 99.40 0.07 1634467.33 0.10 0.00 61.13 0.20 1.57 ... 0.10 0.13 0.13 0.13 0.17 0.07 0.00 0.53 0.63 1.30
14 845900.93 0.03 99.48 0.03 1657039.87 0.13 0.03 61.53 0.07 0.77 ... 0.07 0.20 0.07 0.00 0.10 0.00 0.00 0.03 0.10 1.57
15 882909.60 0.27 101.85 0.10 1784909.47 0.10 0.00 61.53 0.30 0.10 ... 0.13 0.13 0.27 0.17 0.17 1.13 0.00 0.17 0.23 1.19
16 933405.87 0.07 98.61 0.07 2098409.40 0.03 0.07 59.80 0.03 0.03 ... 0.00 0.33 2.20 0.00 0.00 0.00 0.03 2.42 0.07 0.00
17 1066830.67 0.17 97.44 0.10 2146756.47 0.13 0.03 61.40 0.10 0.53 ... 0.10 0.20 9.01 0.07 0.53 0.10 0.00 0.60 0.00 0.17
18 1118026.60 4.33 96.77 0.03 2180189.60 0.07 0.00 60.67 0.13 0.10 ... 0.00 0.13 5.87 0.00 0.43 0.67 0.00 0.43 0.10 0.00
19 1152583.60 0.17 94.59 0.27 2202088.13 0.13 0.00 60.93 0.17 0.43 ... 0.13 0.20 0.17 0.13 0.77 0.37 0.00 0.60 0.17 0.10
20 1197990.67 0.03 91.11 0.27 2442916.07 0.17 0.03 61.07 0.10 0.13 ... 0.03 0.10 0.10 0.00 0.83 0.00 0.00 0.33 0.20 0.03
21 1241711.20 0.07 100.45 0.37 2656067.60 0.27 0.03 60.27 0.23 0.40 ... 0.07 0.17 0.10 0.13 1.03 0.27 0.03 0.20 0.13 0.10
22 1280063.27 0.07 94.57 0.27 2679043.27 0.00 0.03 60.73 0.00 0.03 ... 0.10 0.00 0.77 0.00 0.67 0.00 0.03 0.00 0.60 0.00
23 1327774.67 1.27 92.39 0.43 2700150.07 0.13 0.00 61.13 0.10 0.20 ... 0.20 0.23 1.00 0.10 0.20 1.13 0.07 0.30 0.27 0.10
24 1371158.87 0.00 96.58 0.20 2910037.73 0.13 0.00 61.80 0.03 0.13 ... 0.00 0.10 0.03 0.07 0.17 1.13 0.00 0.03 0.03 0.03
25 1422435.87 0.20 97.98 0.67 3151953.07 0.10 0.00 60.73 0.60 0.20 ... 0.20 0.17 0.10 0.07 0.97 0.70 0.00 0.07 0.47 0.07
26 1469376.40 0.03 93.07 0.37 3183915.00 0.00 0.20 61.73 0.43 0.03 ... 0.00 0.00 0.03 0.00 0.00 0.00 0.00 0.37 0.43 0.03
27 1530692.33 0.13 NaN 0.40 3228108.00 0.20 0.00 NaN 0.57 0.17 ... 0.10 0.13 0.30 0.17 0.20 0.07 0.00 0.50 0.17 0.49
28 1573513.40 0.00 96.61 0.20 3392034.73 0.00 0.03 61.07 0.57 0.03 ... 0.00 0.13 0.03 0.10 0.03 0.00 0.03 0.07 0.03 0.00
29 1617539.47 0.17 98.70 0.37 3689794.93 0.47 0.07 61.80 0.77 0.23 ... 0.10 0.17 0.27 0.20 0.20 0.10 0.00 0.53 0.23 0.20
30 1651111.27 0.03 92.99 0.23 3710931.53 0.07 1.13 61.67 0.50 0.03 ... 0.03 0.20 0.07 0.17 0.03 0.03 0.03 1.17 0.07 0.00
31 1684617.27 0.67 96.79 0.33 3732230.80 0.57 0.00 61.93 0.13 0.17 ... 0.23 0.17 0.13 0.13 0.13 0.03 0.03 1.33 0.20 0.17
32 1724126.60 0.10 98.77 0.23 3866876.33 0.27 1.39 61.60 0.03 0.33 ... 0.10 0.17 0.03 26.17 1.13 0.07 0.00 0.07 0.03 0.00
33 1785718.60 0.40 98.34 0.37 4203054.47 0.17 0.33 61.73 0.33 0.20 ... 0.20 0.17 5.37 87.01 1.23 0.10 0.00 0.13 0.27 0.13
34 2215138.40 1.30 100.00 0.20 4622494.07 0.00 0.00 61.33 0.10 0.07 ... 0.03 0.07 8.62 91.35 1.17 0.03 0.00 0.07 0.10 0.07
35 2370608.13 5.50 95.88 0.27 4728866.33 0.23 1.05 61.47 0.33 0.60 ... 0.17 0.53 7.93 87.05 0.63 0.17 0.00 0.23 0.57 0.03
36 2411135.27 8.93 97.90 0.23 4754849.27 0.03 0.00 61.60 0.13 0.30 ... 0.07 0.23 0.07 91.07 0.03 0.00 0.00 0.10 0.03 0.03
37 2474337.27 9.03 101.29 0.13 4801485.80 0.07 0.07 61.47 0.17 0.17 ... 0.20 1.37 0.20 87.05 0.07 0.07 0.00 0.20 0.07 0.13
38 2559262.33 0.17 98.60 0.13 5189849.87 0.10 0.10 61.53 0.10 0.13 ... 0.13 1.50 0.03 89.91 0.07 0.07 0.00 0.03 0.27 0.07
39 2625164.80 0.10 100.62 0.17 5320943.40 0.10 0.00 61.33 0.13 0.47 ... 0.07 1.00 0.10 71.62 0.17 0.10 0.00 0.13 0.20 0.07
40 2682692.93 0.27 95.69 0.20 5360812.00 0.23 0.00 60.00 0.17 1.43 ... 0.10 0.23 0.20 0.20 0.10 0.10 0.00 0.10 0.10 0.17
41 2734614.07 0.10 100.71 0.10 5401386.80 0.27 0.07 60.40 0.20 1.37 ... 0.13 0.10 0.07 0.07 0.13 0.03 0.00 0.23 0.17 0.53
42 2807601.20 0.13 99.93 0.13 5752028.53 0.17 0.07 60.53 0.10 0.17 ... 0.17 0.10 0.13 0.17 0.80 0.13 0.00 0.07 0.50 0.70
43 2850531.40 0.03 84.01 0.07 5890009.20 0.07 0.10 59.27 0.03 0.10 ... 0.00 0.03 0.07 0.00 0.23 0.00 0.00 0.03 0.40 0.60
44 2888147.73 0.30 100.14 0.10 5913489.87 0.20 0.07 60.33 0.20 0.10 ... 0.47 0.13 0.10 0.13 0.17 0.46 0.00 0.03 0.67 0.70
45 2918244.53 0.07 96.91 0.13 5929892.40 0.07 0.00 60.80 0.10 0.33 ... 0.10 0.67 0.07 0.03 0.23 1.13 0.03 0.10 0.13 0.27
46 2961105.53 0.77 96.78 0.57 6231183.20 0.23 0.00 61.93 1.13 0.10 ... 0.10 0.13 0.20 0.13 0.27 0.07 0.00 0.23 0.40 1.07

47 rows × 154 columns

Warning: Total number of columns (154) exceeds max_columns (20) limiting to first (20) columns.
In [ ]:
pj = api.from_path("utensil/falcon-qlora")
In [113]:
runs = api.runs(path="utensil/falcon-qlora")
In [116]:
len(runs)
Out[116]:
46
In [120]:
for run in runs:
  print(run.name)

twilight-wildflower-46
solar-violet-45
treasured-shadow-44
visionary-frog-43
bumbling-butterfly-42
classic-monkey-41
cool-yogurt-40
misunderstood-leaf-39
dashing-resonance-38
royal-morning-37
still-resonance-36
serene-spaceship-35
rich-snowball-34
effortless-night-33
northern-totem-32
morning-wind-31
sparkling-bee-30
warm-surf-29
fresh-eon-28
ethereal-blaze-27
revived-meadow-26
stoic-water-25
clear-fire-24
ruby-snowball-23
breezy-night-22
dutiful-night-21
run-20: 40b 4*2 + xformer
run-19: 40b 4*2 + xformer
run-18: 40b 1*2 + xformer
run-17: 7b 40*2 + xformer
run-16: 64*2 + xformer OOM
run-15: 32*2 + xformer
run-14: 16*2 - no max len
run-13: 16*2 - no max len
run-12: 4*2 - no max len
run-11: 4/8 cont.
run-10: 7b 4*2 + mpsl 2048 
run-9: 7b 4/8 DISK FULL
run-8: 4/8
run-7: 7b 1/1
run-6: 12/24
run-5: 12/24 OOM
run-4: 10/20
run-3: 8/16
run-2: 6/12
run-1: 1b 4*2
In [ ]:
!pip install genv

Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/
Collecting genv
  Downloading genv-1.0.0-py3-none-any.whl (69 kB)
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 69.2/69.2 kB 2.6 MB/s eta 0:00:00
Installing collected packages: genv
Successfully installed genv-1.0.0
In [ ]:
!eval "$(genv shell --init)"
In [ ]:
!genv activate

Your shell is not properly initialized at the moment.
Run the following command to initialize it.
You should also add it to your ~/.bashrc or any equivalent file.

    eval "$(genv shell --init)"
In [ ]:
!genv config name mp

Not running in an active environment
In [ ]:
!pip install colab-xterm
%load_ext colabxterm

Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/
Collecting colab-xterm
  Downloading colab_xterm-0.1.2-py3-none-any.whl (115 kB)
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 0.0/115.3 kB ? eta -:--:--
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 115.3/115.3 kB 4.6 MB/s eta 0:00:00
Requirement already satisfied: ptyprocess~=0.7.0 in /usr/local/lib/python3.10/dist-packages (from colab-xterm) (0.7.0)
Requirement already satisfied: tornado>5.1 in /usr/local/lib/python3.10/dist-packages (from colab-xterm) (6.3.1)
Installing collected packages: colab-xterm
Successfully installed colab-xterm-0.1.2
In [ ]:
%cd /content/

/content
In [ ]:
%xterm

Launching Xterm...
In [ ]:
!pip install tensorflow
!pip install ai-benchmark

Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/
Requirement already satisfied: tensorflow in /usr/local/lib/python3.10/dist-packages (2.12.0)
Requirement already satisfied: absl-py>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (1.4.0)
Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (1.6.3)
Requirement already satisfied: flatbuffers>=2.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (23.3.3)
Requirement already satisfied: gast<=0.4.0,>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (0.4.0)
Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (0.2.0)
Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (1.54.0)
Requirement already satisfied: h5py>=2.9.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (3.8.0)
Requirement already satisfied: jax>=0.3.15 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (0.4.10)
Requirement already satisfied: keras<2.13,>=2.12.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (2.12.0)
Requirement already satisfied: libclang>=13.0.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (16.0.0)
Requirement already satisfied: numpy<1.24,>=1.22 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (1.22.4)
Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (3.3.0)
Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from tensorflow) (23.1)
Requirement already satisfied: protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (3.20.3)
Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from tensorflow) (67.7.2)
Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (1.16.0)
Requirement already satisfied: tensorboard<2.13,>=2.12 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (2.12.2)
Requirement already satisfied: tensorflow-estimator<2.13,>=2.12.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (2.12.0)
Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (2.3.0)
Requirement already satisfied: typing-extensions>=3.6.6 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (4.5.0)
Requirement already satisfied: wrapt<1.15,>=1.11.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (1.14.1)
Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (0.32.0)
Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from astunparse>=1.6.0->tensorflow) (0.40.0)
Requirement already satisfied: ml-dtypes>=0.1.0 in /usr/local/lib/python3.10/dist-packages (from jax>=0.3.15->tensorflow) (0.1.0)
Requirement already satisfied: scipy>=1.7 in /usr/local/lib/python3.10/dist-packages (from jax>=0.3.15->tensorflow) (1.10.1)
Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.13,>=2.12->tensorflow) (2.17.3)
Requirement already satisfied: google-auth-oauthlib<1.1,>=0.5 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.13,>=2.12->tensorflow) (1.0.0)
Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.13,>=2.12->tensorflow) (3.4.3)
Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.13,>=2.12->tensorflow) (2.27.1)
Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.13,>=2.12->tensorflow) (0.7.0)
Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.13,>=2.12->tensorflow) (1.8.1)
Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.13,>=2.12->tensorflow) (2.3.0)
Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.13,>=2.12->tensorflow) (5.3.0)
Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.13,>=2.12->tensorflow) (0.3.0)
Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.13,>=2.12->tensorflow) (4.9)
Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from google-auth-oauthlib<1.1,>=0.5->tensorboard<2.13,>=2.12->tensorflow) (1.3.1)
Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.21.0->tensorboard<2.13,>=2.12->tensorflow) (1.26.15)
Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.21.0->tensorboard<2.13,>=2.12->tensorflow) (2022.12.7)
Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.21.0->tensorboard<2.13,>=2.12->tensorflow) (2.0.12)
Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.21.0->tensorboard<2.13,>=2.12->tensorflow) (3.4)
Requirement already satisfied: MarkupSafe>=2.1.1 in /usr/local/lib/python3.10/dist-packages (from werkzeug>=1.0.1->tensorboard<2.13,>=2.12->tensorflow) (2.1.2)
Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.13,>=2.12->tensorflow) (0.5.0)
Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.10/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<1.1,>=0.5->tensorboard<2.13,>=2.12->tensorflow) (3.2.2)
Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/
Requirement already satisfied: ai-benchmark in /usr/local/lib/python3.10/dist-packages (0.1.2)
Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from ai-benchmark) (1.22.4)
Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from ai-benchmark) (5.9.5)
Requirement already satisfied: py-cpuinfo in /usr/local/lib/python3.10/dist-packages (from ai-benchmark) (9.0.0)
Requirement already satisfied: pillow in /usr/local/lib/python3.10/dist-packages (from ai-benchmark) (8.4.0)
Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from ai-benchmark) (67.7.2)
Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from ai-benchmark) (2.27.1)
Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->ai-benchmark) (1.26.15)
Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->ai-benchmark) (2022.12.7)
Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->ai-benchmark) (2.0.12)
Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->ai-benchmark) (3.4)
In [ ]:
from ai_benchmark import AIBenchmark
benchmark = AIBenchmark()
results = benchmark.run()

>>   AI-Benchmark-v.0.1.2   
>>   Let the AI Games begin..

*  TF Version: 2.12.0
*  Platform: Linux-5.15.107+-x86_64-with-glibc2.31
*  CPU: N/A
*  CPU RAM: 13 GB
*  GPU/0: Tesla T4
*  GPU RAM: 13.3 GB
*  CUDA Version: 11.8
*  CUDA Build: V11.8.89

The benchmark is running...
The tests might take up to 20 minutes
Please don't interrupt the script

1/19. MobileNet-V2

1.1 - inference | batch=50, size=224x224: 80.3 ± 1.5 ms
1.2 - training  | batch=50, size=224x224: 209 ± 2 ms

2/19. Inception-V3

2.1 - inference | batch=20, size=346x346: 103 ± 2 ms
2.2 - training  | batch=20, size=346x346: 396 ± 248 ms

3/19. Inception-V4

3.1 - inference | batch=10, size=346x346: 98.6 ± 3.5 ms
3.2 - training  | batch=10, size=346x346: 356 ± 5 ms

4/19. Inception-ResNet-V2

4.1 - inference | batch=10, size=346x346: 131 ± 5 ms
4.2 - training  | batch=8, size=346x346: 367 ± 2 ms

5/19. ResNet-V2-50

5.1 - inference | batch=10, size=346x346: 74.7 ± 8.3 ms

---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-100-eb3ed2f4b010> in <cell line: 3>()
      1 from ai_benchmark import AIBenchmark
      2 benchmark = AIBenchmark()
----> 3 results = benchmark.run()

/usr/local/lib/python3.10/dist-packages/ai_benchmark/__init__.py in run(self, precision)
     61 
     62     def run(self, precision="normal"):
---> 63         return run_tests(training=True, inference=True, micro=False, verbose=self.verbose,
     64                          use_CPU=self.use_CPU, precision=precision, _type="full", start_dir=self.cwd)
     65 

/usr/local/lib/python3.10/dist-packages/ai_benchmark/utils.py in run_tests(training, inference, micro, verbose, use_CPU, precision, _type, start_dir)
    633 
    634                             time_iter_started = getTimeMillis()
--> 635                             sess.run(train_step, feed_dict={input_: data, target_: target})
    636                             training_time = getTimeMillis() - time_iter_started
    637                             training_times.append(training_time)

/usr/local/lib/python3.10/dist-packages/tensorflow/python/client/session.py in run(self, fetches, feed_dict, options, run_metadata)
    966 
    967     try:
--> 968       result = self._run(None, fetches, feed_dict, options_ptr,
    969                          run_metadata_ptr)
    970       if run_metadata:

/usr/local/lib/python3.10/dist-packages/tensorflow/python/client/session.py in _run(self, handle, fetches, feed_dict, options, run_metadata)
   1189     # or if the call is a partial run that specifies feeds.
   1190     if final_fetches or final_targets or (handle and feed_dict_tensor):
-> 1191       results = self._do_run(handle, final_targets, final_fetches,
   1192                              feed_dict_tensor, options, run_metadata)
   1193     else:

/usr/local/lib/python3.10/dist-packages/tensorflow/python/client/session.py in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
   1369 
   1370     if handle is None:
-> 1371       return self._do_call(_run_fn, feeds, fetches, targets, options,
   1372                            run_metadata)
   1373     else:

/usr/local/lib/python3.10/dist-packages/tensorflow/python/client/session.py in _do_call(self, fn, *args)
   1376   def _do_call(self, fn, *args):
   1377     try:
-> 1378       return fn(*args)
   1379     except errors.OpError as e:
   1380       message = compat.as_text(e.message)

/usr/local/lib/python3.10/dist-packages/tensorflow/python/client/session.py in _run_fn(feed_dict, fetch_list, target_list, options, run_metadata)
   1359       # Ensure any changes to the graph are reflected in the runtime.
   1360       self._extend_graph()
-> 1361       return self._call_tf_sessionrun(options, feed_dict, fetch_list,
   1362                                       target_list, run_metadata)
   1363 

/usr/local/lib/python3.10/dist-packages/tensorflow/python/client/session.py in _call_tf_sessionrun(self, options, feed_dict, fetch_list, target_list, run_metadata)
   1452   def _call_tf_sessionrun(self, options, feed_dict, fetch_list, target_list,
   1453                           run_metadata):
-> 1454     return tf_session.TF_SessionRun_wrapper(self._session, options, feed_dict,
   1455                                             fetch_list, target_list,
   1456                                             run_metadata)

KeyboardInterrupt:
In [ ]: