# DDPG's way of value estimation def get_qval(self, observation, action): """Get Q-val.""" return self.critic.get_qval(observation, action) # TD3 and C2A2's way of value estimation def get_qval(self, observation, action): """Get Q-val.""" q_val1 = self.critic1.get_qval(observation, action) q_val2 = self.critic2.get_qval(observation, action) return np.minimum(q_val1, q_val2) # DDPG and TD3's way for action selection def action(self, observation): """Return an action according to the agent's policy.""" return self.actor.get_action(observation) # C2A2's way for action selection def action(self, observation): """Return an action according to the agent's policy.""" action1 = self.actor1.get_action(observation) action2 = self.actor2.get_action(observation) q_val1 = self.get_qval(observation, action1) q_val2 = self.get_qval(observation, action2) q_val1 = np.expand_dims(q_val1, axis=-1) q_val2 = np.expand_dims(q_val2, axis=-1) return np.where(q_val1 > q_val2, action1, action2)
# DDPG's way of value estimation
def get_qval(self, observation, action):
"""Get Q-val."""
return self.critic.get_qval(observation, action)
# TD3 and C2A2's way of value estimation
q_val1 = self.critic1.get_qval(observation, action)
q_val2 = self.critic2.get_qval(observation, action)
return np.minimum(q_val1, q_val2)
# DDPG and TD3's way for action selection
def action(self, observation):
"""Return an action according to the agent's policy."""
return self.actor.get_action(observation)
# C2A2's way for action selection
action1 = self.actor1.get_action(observation)
action2 = self.actor2.get_action(observation)
q_val1 = self.get_qval(observation, action1)
q_val2 = self.get_qval(observation, action2)
q_val1 = np.expand_dims(q_val1, axis=-1)
q_val2 = np.expand_dims(q_val2, axis=-1)
return np.where(q_val1 > q_val2, action1, action2)
import gym env = gym.make('BipedalWalker-v2') obs = env.reset()
import gym
env = gym.make('BipedalWalker-v2')
obs = env.reset()
agent = Agent(...) done = False while not done: action = agent.take_action(obs) obs, reward, done , info = env.step(action)
agent = Agent(...)
done = False
while not done:
action = agent.take_action(obs)
obs, reward, done , info = env.step(action)
hyperparameter.yaml
gcloud ml engine submit
gym
import argparse parser = argparse.ArgumentParser() ... parser.add_argument( '--batch_size', help = 'Batch size for agent.', type = int, default = 64 ) ... args = parser.parse_args() run_model(args)
import argparse
parser = argparse.ArgumentParser()
...
parser.add_argument(
'--batch_size',
help = 'Batch size for agent.',
type = int,
default = 64
)
args = parser.parse_args()
run_model(args)
Model
metrics
model = Model(...) ... model.compile(loss='rmse', metrics=['rmse'])
model = Model(...)
model.compile(loss='rmse', metrics=['rmse'])
# Initialize a tensor. import keras.backend as K reward_tensor = K.variable(value=0) # Create custom reward. K.get_session().run(tf.global_variables_initializer()) def reward(y_true, y_pred): return self.reward_tensor model.compile(loss='rmse', metrics=reward]) # At each iteration, update tensor: for _ in range(nepochs): ... K.set_value(self.reward_tensor, self.cur_reward) ...
# Initialize a tensor.
import keras.backend as K
reward_tensor = K.variable(value=0)
# Create custom reward.
K.get_session().run(tf.global_variables_initializer())
def reward(y_true, y_pred):
return self.reward_tensor
model.compile(loss='rmse', metrics=reward])
# At each iteration, update tensor:
for _ in range(nepochs):
K.set_value(self.reward_tensor, self.cur_reward)
episode_reward = tf.placeholder(dtype=tf.float32, shape=[]) tf.summary.scalar('reward', episode_reward) with tf.Session() as sess: ... merge = tf.summary.merge_all() summary = sess.run(merge, feed_dict={episode_reward: reward}) train_writer.add_summary(summary, iteration) ...
episode_reward = tf.placeholder(dtype=tf.float32, shape=[])
tf.summary.scalar('reward', episode_reward)
with tf.Session() as sess:
merge = tf.summary.merge_all()
summary = sess.run(merge, feed_dict={episode_reward: reward})
train_writer.add_summary(summary, iteration)
reward
--job-dir
TF_CONFIG
# Append trial_id to path if we are doing hp tuning. trial_id = json.loads( os.environ.get('TF_CONFIG', '{}') ).get('task', {}).get('trial', '') model_dir_hp = os.path.join(model_dir, trial_id)
# Append trial_id to path if we are doing hp tuning.
trial_id = json.loads(
os.environ.get('TF_CONFIG', '{}')
).get('task', {}).get('trial', '')
model_dir_hp = os.path.join(model_dir, trial_id)
__init__.py
setup.py
rl_on_gcp # root directory setup.py # says how to install your package trainer # package name, “trainer” is convention # .py files specific to your job, e.g. model.py __init__.py # python convention indicating this is a package
rl_on_gcp # root directory
setup.py # says how to install your package
trainer # package name, “trainer” is convention
# .py files specific to your job, e.g. model.py
__init__.py # python convention indicating this is a package
yaml
maxTrials
maxParallelTrials
algorithm
GRID_SEARCH
trainingInput: scaleTier: BASIC_GPU hyperparameters: maxTrials: 40 maxParallelTrials: 5 enableTrialEarlyStopping: False goal: MAXIMIZE hyperparameterMetricTag: reward params: - parameterName: update_target type: INTEGER minValue: 5000 maxValue: 20000 scaleType: UNIT_LOG_SCALE - parameterName: init_eta type: DOUBLE minValue: 0.8 maxValue: 0.95 scaleType: UNIT_LOG_SCALE - parameterName: learning_rate type: DOUBLE minValue: 0.00001 maxValue: 0.001 scaleType: UNIT_LOG_SCALE - parameterName: batch_size type: DISCRETE discreteValues: - 4 - 16 - 32 - 64 - 128 - 256 - 512
trainingInput:
scaleTier: BASIC_GPU
hyperparameters:
maxTrials: 40
maxParallelTrials: 5
enableTrialEarlyStopping: False
goal: MAXIMIZE
hyperparameterMetricTag: reward
params:
- parameterName: update_target
type: INTEGER
minValue: 5000
maxValue: 20000
scaleType: UNIT_LOG_SCALE
- parameterName: init_eta
type: DOUBLE
minValue: 0.8
maxValue: 0.95
- parameterName: learning_rate
minValue: 0.00001
maxValue: 0.001
- parameterName: batch_size
type: DISCRETE
discreteValues:
- 4
- 16
- 32
- 64
- 128
- 256
- 512
JOBNAME=rl_job_hp_$(date -u +%y%m%d_%H%M%S) REGION=us-central1 BUCKET=your-bucket gcloud ml-engine jobs submit training $JOBNAME \ --package-path=$PWD/rl_on_gcp/trainer \ --module-name=trainer.trainer \ --region=$REGION \ --staging-bucket=gs://$BUCKET \ --config=hyperparam.yaml \ --runtime-version=1.10 \ --\ --steps=100000\ --start_train=10000\ --buffer_size=10000\ --model_dir='gs://your-bucket/rl_on_gcp'
JOBNAME=rl_job_hp_$(date -u +%y%m%d_%H%M%S)
REGION=us-central1
BUCKET=your-bucket
gcloud ml-engine jobs submit training $JOBNAME \
--package-path=$PWD/rl_on_gcp/trainer \
--module-name=trainer.trainer \
--region=$REGION \
--staging-bucket=gs://$BUCKET \
--config=hyperparam.yaml \
--runtime-version=1.10 \
--\
--steps=100000\
--start_train=10000\
--buffer_size=10000\
--model_dir='gs://your-bucket/rl_on_gcp'
--config
"discount_rate": "0.83375867407642434", "n_games_per_update": "4", "n_hidden": "16", "learning_rate": "0.055623309804870541"
"discount_rate": "0.83375867407642434",
"n_games_per_update": "4",
"n_hidden": "16",
"learning_rate": "0.055623309804870541"
Google Compute Engine、Google Cloud Storage、Google BigQuery、Stackdriver Logging、Stackdriver Monitoring、Google Cloud Functions、Cloud Pub/Sub、Virtual Private Cloud（VPC）、Cloud VPN など
フォスター電機株式会社1949 年の創業以来、「音のスペシャリスト」として、スピーカ事業、モバイル オーディオ事業を主力に、小型音響部品などの事業を加えた 3 つの事業を推進する音響メーカー。スピーカやヘッドホン、警報音用ブザーなどを OEM 生産、ODM 生産するほか、自社ブランド「Fostex」を展開。現在は、車載用スピーカに注力しており、日本、欧州、米国などの主要な自動車メーカーと取引をしている。 株式会社トップゲート(Google Cloud Platform パートナー)
Google Kubernetes Engine、Cloud Firestore（ベータ版）、Cloud Scheduler（ベータ版）、Google Stackdriver、Google BigQuery など
株式会社Ginco は「次世代の銀行をつくる」を理念に、仮想通貨ウォレット アプリ「Ginco」を提供するブロックチェーン事業者。その他にも、仮想通貨のマイニング事業や、法人向けのブロックチェーン サービス開発支援事業を行い、仮想通貨のウォレットという "入口" から、それをどう活用していくかという "出口" の部分までを、ブロックチェーン技術を基盤に広く展開している。2017 年 12 月設立、従業員数は 約 30 名（2018 年 12 月時点）
1990 年 9 月の設立以来、医療システム、写真プリント システム、デジタル カメラ、印刷システムなど、富士フイルムの主力システム製品の企画から運用保守まで、製品のライフ サイクル全体にわたるソフトウエアの開発を事業として展開。
Google Compute Engine、Google Kubernetes Engine、Google Cloud Storage、Google Stackdriver、Google BigQuery、Cloud SQL、Cloud Pub/Sub、Google Cloud Functions など
