AutoModel - HyperParameter Search

In [ ]:

import os

# you can remove this
os.chdir("../../")

import os # you can remove this os.chdir("../../")

In [ ]:

from gradsflow import Model
from gradsflow.tuner.tuner import Tuner

from gradsflow import Model from gradsflow.tuner.tuner import Tuner

In [ ]:

from timm import create_model
from gradsflow.data.image import get_fake_data

from gradsflow import AutoDataset

from timm import create_model from gradsflow.data.image import get_fake_data from gradsflow import AutoDataset

In [ ]:

image_size = (128, 128)
fake_data = get_fake_data(image_size, num_workers=0)
train_ds, train_dl = fake_data['ds'], fake_data['dl']

fake_data = get_fake_data(image_size, num_workers=0)
val_ds, val_dl = fake_data['ds'], fake_data['dl']

num_classes = train_ds.num_classes
autodataset = AutoDataset(train_dl, val_dl, num_classes=num_classes)

image_size = (128, 128) fake_data = get_fake_data(image_size, num_workers=0) train_ds, train_dl = fake_data['ds'], fake_data['dl'] fake_data = get_fake_data(image_size, num_workers=0) val_ds, val_dl = fake_data['ds'], fake_data['dl'] num_classes = train_ds.num_classes autodataset = AutoDataset(train_dl, val_dl, num_classes=num_classes)

In [ ]:

from gradsflow.tuner.tuner import Tuner
from gradsflow.tuner.automodel import AutoModelV2

from gradsflow.tuner.tuner import Tuner from gradsflow.tuner.automodel import AutoModelV2

In [ ]:

cnn1 = create_model("resnet18", pretrained=False, num_classes=num_classes)
cnn2 = create_model("efficientnet_b0", pretrained=False, num_classes=num_classes)

cnn1 = create_model("resnet18", pretrained=False, num_classes=num_classes) cnn2 = create_model("efficientnet_b0", pretrained=False, num_classes=num_classes)

In [ ]:

tuner = Tuner()

cnns = tuner.suggest_complex("learner", cnn1, cnn2)
optimizers = tuner.choice("optimizer", "adam", "sgd")
loss = tuner.choice(
    "loss",
    "crossentropyloss",
)

model = AutoModelV2(cnns)

tuner = Tuner() cnns = tuner.suggest_complex("learner", cnn1, cnn2) optimizers = tuner.choice("optimizer", "adam", "sgd") loss = tuner.choice( "loss", "crossentropyloss", ) model = AutoModelV2(cnns)

In [ ]:

model.hp_tune(tuner, autodataset, epochs=4)

model.hp_tune(tuner, autodataset, epochs=4)

2021-09-29 19:43:53,414	INFO services.py:1263 -- View the Ray dashboard at http://127.0.0.1:8265
2021-09-29 19:43:56,265	WARNING function_runner.py:558 -- Function checkpointing is disabled. This may result in unexpected behavior when using checkpointing features or certain schedulers. To enable, set the train function arguments to be `func(config, checkpoint_dir=None)`.
2021-09-29 19:43:58,540	WARNING worker.py:1215 -- Warning: The actor ImplicitFunc is very large (58 MiB). Check that its definition is not implicitly capturing a large array or other object in scope. Tip: use ray.put() to put large objects in the Ray object store.

== Status ==
Memory usage on this node: 10.9/16.0 GiB
Using FIFO scheduling algorithm.
Resources requested: 1.0/8 CPUs, 0/0 GPUs, 0.0/4.38 GiB heap, 0.0/2.19 GiB objects
Result logdir: /Users/aniket/ray_results/_hp_optimizer_2021-09-29_19-43-56
Number of trials: 1/1 (1 RUNNING)

Trial name	status	loc	learner	loss	optimizer
_hp_optimizer_7c10a_00000	RUNNING		1	crossentropyloss	sgd

Result for _hp_optimizer_7c10a_00000:
  date: 2021-09-29_19-44-38
  done: false
  experiment_id: 1f6a6823b4544df88fecde2ffef3b208
  hostname: Anikets-Turing-Machine.local
  iterations_since_restore: 1
  node_ip: 192.168.120.84
  pid: 2207
  should_checkpoint: true
  time_since_restore: 33.17194175720215
  time_this_iter_s: 33.17194175720215
  time_total_s: 33.17194175720215
  timestamp: 1632924878
  timesteps_since_restore: 0
  train_loss: 2.4102004801986
  training_iteration: 1
  trial_id: 7c10a_00000
  val_accuracy: 0.11
  val_loss: 285.92313784330526
  

== Status ==
Memory usage on this node: 9.9/16.0 GiB
Using FIFO scheduling algorithm.
Resources requested: 1.0/8 CPUs, 0/0 GPUs, 0.0/4.38 GiB heap, 0.0/2.19 GiB objects
Current best trial: 7c10a_00000 with val_loss=285.92313784330526 and parameters={'learner': 1, 'optimizer': 'sgd', 'loss': 'crossentropyloss'}
Result logdir: /Users/aniket/ray_results/_hp_optimizer_2021-09-29_19-43-56
Number of trials: 1/1 (1 RUNNING)

Trial name	status	loc	learner	loss	optimizer	iter	total time (s)	val_loss	val_accuracy	train_loss
_hp_optimizer_7c10a_00000	RUNNING	192.168.120.84:2207	1	crossentropyloss	sgd	1	33.1719	285.923	0.11	2.4102

Result for _hp_optimizer_7c10a_00000:
  date: 2021-09-29_19-45-11
  done: false
  experiment_id: 1f6a6823b4544df88fecde2ffef3b208
  hostname: Anikets-Turing-Machine.local
  iterations_since_restore: 2
  node_ip: 192.168.120.84
  pid: 2207
  should_checkpoint: true
  time_since_restore: 66.14079689979553
  time_this_iter_s: 32.968855142593384
  time_total_s: 66.14079689979553
  timestamp: 1632924911
  timesteps_since_restore: 0
  train_loss: 2.2884897565612823
  training_iteration: 2
  trial_id: 7c10a_00000
  val_accuracy: 0.11
  val_loss: 104.01406013601054
  

== Status ==
Memory usage on this node: 9.7/16.0 GiB
Using FIFO scheduling algorithm.
Resources requested: 1.0/8 CPUs, 0/0 GPUs, 0.0/4.38 GiB heap, 0.0/2.19 GiB objects
Current best trial: 7c10a_00000 with val_loss=104.01406013601054 and parameters={'learner': 1, 'optimizer': 'sgd', 'loss': 'crossentropyloss'}
Result logdir: /Users/aniket/ray_results/_hp_optimizer_2021-09-29_19-43-56
Number of trials: 1/1 (1 RUNNING)

Trial name	status	loc	learner	loss	optimizer	iter	total time (s)	val_loss	val_accuracy	train_loss
_hp_optimizer_7c10a_00000	RUNNING	192.168.120.84:2207	1	crossentropyloss	sgd	2	66.1408	104.014	0.11	2.28849

2021-09-29 19:45:12,050	WARNING util.py:163 -- The `process_trial_save` operation took 0.629 s, which may be a performance bottleneck.

Result for _hp_optimizer_7c10a_00000:
  date: 2021-09-29_19-45-39
  done: false
  experiment_id: 1f6a6823b4544df88fecde2ffef3b208
  hostname: Anikets-Turing-Machine.local
  iterations_since_restore: 3
  node_ip: 192.168.120.84
  pid: 2207
  should_checkpoint: true
  time_since_restore: 94.64873600006104
  time_this_iter_s: 28.507939100265503
  time_total_s: 94.64873600006104
  timestamp: 1632924939
  timesteps_since_restore: 0
  train_loss: 2.2546736681235866
  training_iteration: 3
  trial_id: 7c10a_00000
  val_accuracy: 0.07
  val_loss: 165.44398209004063
  

== Status ==
Memory usage on this node: 10.1/16.0 GiB
Using FIFO scheduling algorithm.
Resources requested: 1.0/8 CPUs, 0/0 GPUs, 0.0/4.38 GiB heap, 0.0/2.19 GiB objects
Current best trial: 7c10a_00000 with val_loss=165.44398209004063 and parameters={'learner': 1, 'optimizer': 'sgd', 'loss': 'crossentropyloss'}
Result logdir: /Users/aniket/ray_results/_hp_optimizer_2021-09-29_19-43-56
Number of trials: 1/1 (1 RUNNING)

Trial name	status	loc	learner	loss	optimizer	iter	total time (s)	val_loss	val_accuracy	train_loss
_hp_optimizer_7c10a_00000	RUNNING	192.168.120.84:2207	1	crossentropyloss	sgd	3	94.6487	165.444	0.07	2.25467

Result for _hp_optimizer_7c10a_00000:
  date: 2021-09-29_19-46-08
  done: false
  experiment_id: 1f6a6823b4544df88fecde2ffef3b208
  hostname: Anikets-Turing-Machine.local
  iterations_since_restore: 4
  node_ip: 192.168.120.84
  pid: 2207
  should_checkpoint: true
  time_since_restore: 123.15836071968079
  time_this_iter_s: 28.50962471961975
  time_total_s: 123.15836071968079
  timestamp: 1632924968
  timesteps_since_restore: 0
  train_loss: 2.237922847248586
  training_iteration: 4
  trial_id: 7c10a_00000
  val_accuracy: 0.08
  val_loss: 59.70871727347269
  

== Status ==
Memory usage on this node: 10.0/16.0 GiB
Using FIFO scheduling algorithm.
Resources requested: 1.0/8 CPUs, 0/0 GPUs, 0.0/4.38 GiB heap, 0.0/2.19 GiB objects
Current best trial: 7c10a_00000 with val_loss=59.70871727347269 and parameters={'learner': 1, 'optimizer': 'sgd', 'loss': 'crossentropyloss'}
Result logdir: /Users/aniket/ray_results/_hp_optimizer_2021-09-29_19-43-56
Number of trials: 1/1 (1 RUNNING)

Trial name	status	loc	learner	loss	optimizer	iter	total time (s)	val_loss	val_accuracy	train_loss
_hp_optimizer_7c10a_00000	RUNNING	192.168.120.84:2207	1	crossentropyloss	sgd	4	123.158	59.7087	0.08	2.23792

Result for _hp_optimizer_7c10a_00000:
  date: 2021-09-29_19-46-08
  done: true
  experiment_id: 1f6a6823b4544df88fecde2ffef3b208
  experiment_tag: 0_learner=1,loss=crossentropyloss,optimizer=sgd
  hostname: Anikets-Turing-Machine.local
  iterations_since_restore: 4
  node_ip: 192.168.120.84
  pid: 2207
  should_checkpoint: true
  time_since_restore: 123.15836071968079
  time_this_iter_s: 28.50962471961975
  time_total_s: 123.15836071968079
  timestamp: 1632924968
  timesteps_since_restore: 0
  train_loss: 2.237922847248586
  training_iteration: 4
  trial_id: 7c10a_00000
  val_accuracy: 0.08
  val_loss: 59.70871727347269
  
(pid=2207) Epoch Progress... ━━━━━━━━━━━━━━━━━━━━━━━━ 100% 0:00:00 ┏━━━━━━━┳━━━━━━━┳━━━━━━┓
(pid=2207)                                                         ┃ epoch ┃ trai… ┃ val… ┃
(pid=2207)                                                         ┡━━━━━━━╇━━━━━━━╇━━━━━━┩
(pid=2207)                                                         │ 3     │  2.2… │  59… │
(pid=2207)                                                         └───────┴───────┴──────┘

2021-09-29 19:46:09,363	WARNING util.py:163 -- The `process_trial_save` operation took 0.501 s, which may be a performance bottleneck.

== Status ==
Memory usage on this node: 10.0/16.0 GiB
Using FIFO scheduling algorithm.
Resources requested: 0/8 CPUs, 0/0 GPUs, 0.0/4.38 GiB heap, 0.0/2.19 GiB objects
Current best trial: 7c10a_00000 with val_loss=59.70871727347269 and parameters={'learner': 1, 'optimizer': 'sgd', 'loss': 'crossentropyloss'}
Result logdir: /Users/aniket/ray_results/_hp_optimizer_2021-09-29_19-43-56
Number of trials: 1/1 (1 TERMINATED)

Trial name	status	loc	learner	loss	optimizer	iter	total time (s)	val_loss	val_accuracy	train_loss
_hp_optimizer_7c10a_00000	TERMINATED		1	crossentropyloss	sgd	4	123.158	59.7087	0.08	2.23792

2021-09-29 19:46:09,481	INFO tune.py:561 -- Total run time: 133.22 seconds (132.27 seconds for the tuning loop).

In [ ]:

model.tuner

model.tuner

Out[ ]:

<gradsflow.tuner.tuner.Tuner at 0x7fbfedb6e340>

---

Last update: September 29, 2021

Comments