训练出错

[txssg]

Traceback (most recent call last):
File "/opt/conda/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1334, in _do_call
return fn(*args)
File "/opt/conda/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1319, in _run_fn
options, feed_dict, fetch_list, target_list, run_metadata)
File "/opt/conda/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1407, in _call_tf_sessionrun
run_metadata)
tensorflow.python.framework.errors_impl.OutOfRangeError: PaddingFIFOQueue '_1_get_batch/batch/padding_fifo_queue' is closed and has insufficient elements (requested 8, current size 0)
[[{{node get_batch/batch}} = QueueDequeueManyV2[component_types=[DT_STRING, DT_FLOAT, DT_INT32, DT_INT32, DT_INT32, DT_INT32], timeout_ms=-1, _device="/job:localhost/replica:0/task:0/device:CPU:0"](get_batch/batch/padding_fifo_queue, get_batch/batch/n)]]
[[{{node tower_0/postprocess_detctions/PyFunc_39/_2365}} = _Recvclient_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device_incarnation=1, tensor_name="edge_8411_tower_0/postprocess_detctions/PyFunc_39", tensor_type=DT_INT64, _device="/job:localhost/replica:0/task:0/device:GPU:0"]]

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "multi_gpu_train.py", line 394, in
train()
File "multi_gpu_train.py", line 373, in train
_, global_stepnp, summary_str = sess.run([train_op, global_step, summary_op])
File "/opt/conda/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 929, in run
run_metadata_ptr)
File "/opt/conda/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1152, in _run
feed_dict_tensor, options, run_metadata)
File "/opt/conda/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1328, in _do_run
run_metadata)
File "/opt/conda/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1348, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.OutOfRangeError: PaddingFIFOQueue '_1_get_batch/batch/padding_fifo_queue' is closed and has insufficient elements (requested 8, current size 0)
[[node get_batch/batch (defined at ../data/io/read_tfrecord_multi_gpu.py:120) = QueueDequeueManyV2[component_types=[DT_STRING, DT_FLOAT, DT_INT32, DT_INT32, DT_INT32, DT_INT32], timeout_ms=-1, _device="/job:localhost/replica:0/task:0/device:CPU:0"](get_batch/batch/padding_fifo_queue, get_batch/batch/n)]]
[[{{node tower_0/postprocess_detctions/PyFunc_39/_2365}} = _Recvclient_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device_incarnation=1, tensor_name="edge_8411_tower_0/postprocess_detctions/PyFunc_39", tensor_type=DT_INT64, _device="/job:localhost/replica:0/task:0/device:GPU:0"]]

Caused by op 'get_batch/batch', defined at:
File "multi_gpu_train.py", line 394, in
train()
File "multi_gpu_train.py", line 149, in train
is_training=True)
File "../data/io/read_tfrecord_multi_gpu.py", line 120, in next_batch
dynamic_pad=True)
File "/opt/conda/lib/python3.6/site-packages/tensorflow/python/util/deprecation.py", line 306, in new_func
return func(*args, **kwargs)
File "/opt/conda/lib/python3.6/site-packages/tensorflow/python/training/input.py", line 1017, in batch
name=name)
File "/opt/conda/lib/python3.6/site-packages/tensorflow/python/training/input.py", line 787, in _batch
dequeued = queue.dequeue_many(batch_size, name=name)
File "/opt/conda/lib/python3.6/site-packages/tensorflow/python/ops/data_flow_ops.py", line 478, in dequeue_many
self._queue_ref, n=n, component_types=self._dtypes, name=name)
File "/opt/conda/lib/python3.6/site-packages/tensorflow/python/ops/gen_data_flow_ops.py", line 3487, in queue_dequeue_many_v2
component_types=component_types, timeout_ms=timeout_ms, name=name)
File "/opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "/opt/conda/lib/python3.6/site-packages/tensorflow/python/util/deprecation.py", line 488, in new_func
return func(*args, **kwargs)
File "/opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 3274, in create_op
op_def=op_def)
File "/opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1770, in init
self._traceback = tf_stack.extract_stack()

OutOfRangeError (see above for traceback): PaddingFIFOQueue '_1_get_batch/batch/padding_fifo_queue' is closed and has insufficient elements (requested 8, current size 0)
[[node get_batch/batch (defined at ../data/io/read_tfrecord_multi_gpu.py:120) = QueueDequeueManyV2[component_types=[DT_STRING, DT_FLOAT, DT_INT32, DT_INT32, DT_INT32, DT_INT32], timeout_ms=-1, _device="/job:localhost/replica:0/task:0/device:CPU:0"](get_batch/batch/padding_fifo_queue, get_batch/batch/n)]]
[[{{node tower_0/postprocess_detctions/PyFunc_39/_2365}} = _Recvclient_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device_incarnation=1, tensor_name="edge_8411_tower_0/postprocess_detctions/PyFunc_39", tensor_type=DT_INT64, _device="/job:localhost/replica:0/task:0/device:GPU:0"]]