zdaiot

404NotFound

TensorFlow踩坑指南

发表于 更新于 分类于 ，
本文字数： 8.9k 阅读时长 ≈ 8 分钟

常用地址

ImageNet数据集预训练模型下载地址

输入数据维度报错

1
tensorflow.python.framework.errors_impl.InvalidArgumentError: input must be 4-dimensional[1,1,461,459,3]

考虑是因为数据集中图片存在png、bmp等格式的图片，并不是纯jpg。

解决方法：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
# -*- coding: utf-8 -*- 
"""该文件解决如下问题：文件夹中所有图片都以jpg结尾，但是有些图片真实类型不是jpg，可能是png或者gif
解决方案为：将png或者bmp图片转化为jpg图片，并按原图名称保存。对于gif，读取每一帧图片，并生成随机独特名称保存。
对于原图，可以选择移动还是删除

对于jpeg4py在Ubuntu下需要 sudo apt-get install libturbojpeg
"""
import imghdr
import os
import struct
import cv2
from PIL import Image, ImageSequence
import uuid
import numpy as np
import multiprocessing as mp
import shutil
import jpeg4py as jpeg

image_path = "./IJCAI_2019_AAAC_train/flower_photos" # 数据路径
sub_dir_exit = True # 图片在各个子文件夹中
dir_format = "jpg"  # 要保存的图片后缀
mv_wrong_jpg = True # 对于错误命名图片，是否移动而不删除
mv_path = './IJCAI_2019_AAAC_train/old' # 移动路径

type_dict = {
    'FFD8FF': 'jpg', '89504E47': 'png', '47494638': 'gif', '49492A00': 'tif',
    '424D': 'bmp', '41433130': 'dwg', '38425053': 'psd', '7B5C727466': 'rtf', '3C3F786D6C': 'xml',
    '68746D6C3E': 'html', '44656C69766572792D646174653A': 'eml', 'CFAD12FEC5FD746F': 'dbx', '2142444E': 'pst',
    'D0CF11E0': 'doc/xls', '5374616E64617264204A': 'mdb', 'FF575043': 'wpd', '252150532D41646F6265': 'ps/eps',
    '255044462D312E': 'pdf', 'AC9EBD8F': 'qdf', 'E3828596': 'pwl', '504B0304': 'zip',
    '52617221': 'rar', '57415645': 'wav', '41564920': 'avi', '2E7261FD': 'ram',
    '2E524D46': 'rm', '000001BA': 'mpg', '000001B3': 'mpg', '6D6F6F76': 'mov', '3026B2758E66CF11': 'asf',
    '4D546864': 'mid'
}


def check_remove_broken(img_path):
    try:
        x = jpeg.JPEG(img_path).decode()  # 只能检测jpg，对于png图片同样报错
        # x = cv2.imread(img_path)          #　报warning，
        return False
    except Exception:
        print('Decoding error:', img_path)
        # os.remove(img_path)
        return True


# 转成16进制字符串
def bytes2hex(bytes):
    num = len(bytes)
    hexstr = u""
    for i in range(num):
        t = u"%x" % bytes[i]
        if len(t) % 2:
            hexstr += u"0"
        hexstr += t
    return hexstr.upper()


# 获得类型
def get_filetype(filename):
    file = open(filename, 'rb')
    ftype = 'unknown'

    for k, v in type_dict.items():
        num_bytes = int(len(k) / 2)  # k为16进制，每一位用4bit存储，而下面的无符号char为8bit，除以2得到应该读取多少个char
        file.seek(0)
        # 'B'为无符号char，struct.unpack按照num_bytes个char的形式读取二进制
        hbytes = struct.unpack('B' * num_bytes, file.read(num_bytes))
        code = bytes2hex(hbytes)
        if code == k:
            ftype = v
            break

    file.close()
    return ftype


def modify_image_formate(image_name, origin_format, dir_format='.jpg'):
    '''修改图片为正确的存储格式

    origin_format:图片的正确格式
    image_name: 待修改的图片的存储路径
    dir_format:　目标格式
    
    '''
    if origin_format == 'png' or origin_format == 'bmp':
        image = cv2.imread(image_name)
        (filename, extension) = os.path.splitext(image_name)  # 区别文件名(含路径)与后缀名
        dir_image_name = filename + dir_format  # 可以按照format形式保存图片
        if mv_wrong_jpg:
            if sub_dir_exit:
                mv_path_ = os.path.join(mv_path, image_name.split('/')[-2]) # 子文件路径
            else:
                mv_path_ = mv_path
            if not os.path.exists(mv_path_):
                os.makedirs(mv_path_)

            shutil.move(image_name, os.path.join(mv_path_, image_name.split('/')[-1]))
        else:
            os.remove(image_name)
        cv2.imwrite(dir_image_name, image)

    elif origin_format == 'gif':
        # using Image
        im = Image.open(image_name)
        iter = ImageSequence.Iterator(im)  # GIF图片流的迭代器
        for frame in iter:
            # frame.save("./frame%d.png" % index) # 可以直接保存为png格式
            frame = frame.convert("RGB")
            frame = cv2.cvtColor(np.asarray(frame), cv2.COLOR_RGB2BGR)
            (filepath, tempfilename) = os.path.split(image_name)  # 区别路径与文件名
            new_image_name = os.path.join(filepath, uuid.uuid4().hex + dir_format)
            cv2.imwrite(new_image_name, frame)
            # print('path:', new_image_name) # 打印路径不能再运行 os.path.join(filepath, uuid.uuid4().hex + dir_format)，得到的不是同一个路径
        if mv_wrong_jpg:
            if sub_dir_exit:
                mv_path_ = os.path.join(mv_path, image_name.split('/')[-2]) #　子文件路径，子函数不可修改全局变量
            else:
                mv_path_ = mv_path
            if not os.path.exists(mv_path_):
                os.makedirs(mv_path_)

            shutil.move(image_name, os.path.join(mv_path_, image_name.split('/')[-1]))
        else:
            os.remove(image_name)

        # # using opencv
        # gif = cv2.VideoCapture(image_name)
        # success, frame = gif.read()
        # while(success):
        #     (filepath, tempfilename) = os.path.split(image_name)
        #     new_image_name = os.path.join(filepath, uuid.uuid4().hex + dir_format)
        #     cv2.imwrite(new_image_name, frame)
        #     # print('path:', new_image_name)
        #     success, frame = gif.read()
        # gif.release()
        # if mv_wrong_jpg:
        #     if sub_dir_exit:
        #         mv_path_ = os.path.join(mv_path, image_name.split('/')[-2]) #　子文件路径
        #     else:
        #         mv_path_ = mv_path
        #     if not os.path.exists(mv_path_):
        #         os.makedirs(mv_path_)
        #     shutil.move(image_name, os.path.join(mv_path_, image_name.split('/')[-1]))
        # else:
        #     os.remove(image_name)


def run(image_full_name):
    image_type = get_filetype(image_full_name)

    # 图片存储格式正确时，跳过当前图片，否则修改图片存储格式
    if image_type is dir_format:
        pass
    else:
        print("Modifing {}, it's right format is: {}.".format(image_full_name, image_type))
        modify_image_formate(image_full_name, origin_format=image_type, dir_format='.jpg')


def get_image_list():
    if sub_dir_exit:
        sub_dirs = os.listdir(image_path)
    else:
        sub_dirs = image_path
    img_list = []
    for sub_dir in sub_dirs:
        print("{}----".format(sub_dir))
        if sub_dir_exit:
            image_names = os.listdir(os.path.join(image_path, sub_dir))
        else:
            image_names = sub_dir

        for image_name in image_names:
            if sub_dir_exit:
                image_full_name = os.path.join(image_path, sub_dir, image_name)
            else:
                image_full_name = os.path.join(sub_dir, image_name)

            img_list.append(image_full_name)
    return img_list


if __name__ == "__main__":
    if mv_wrong_jpg and (not os.path.exists(mv_path)):
        os.makedirs(mv_path) 

    img_list = get_image_list()
    pool = mp.Pool()
    pool.map(run, img_list)
    print('Convert Done!')

    # img_list = get_image_list()
    # res = pool.map(check_remove_broken, img_list)

Cannot assign a device for operation

1
2
tensorflow.python.framework.errors_impl.InvalidArgumentError: Cannot assign a device for operation 'resnet_v2_50/logits/biases': Operation was explicitly assigned to /device:GPU:0 but available devices are [ /job:localhost/replica:0/task:0/device:CPU:0 ]. Make sure the device specification refers to a valid device.
     [[Node: resnet_v2_50/logits/biases = VariableV2[_class=["loc:@resnet_v2_50/logits/biases"], container="", dtype=DT_FLOAT, shape=[110], shared_name="", _device="/device:GPU:0"]()]]

解决方法一

可能是未指定GPU运行，例如在sh文件中，

1
2
3
GPUID=$1
python3 "${SCRIPT_DIR}/run_attacks.py" \
  --gpuid="${GPUID}" \

运行时需要指明GPU

1
bash run_attacks.sh 0

解决方法二

1
2
config = tf.ConfigProto(allow_soft_placement = True)
sess = tf.Session(config = config)

对于slim：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
session_config = tf.ConfigProto(allow_soft_placement=True)

slim.learning.train(
                train_tensor,
                logdir=FLAGS.train_dir,
                master=FLAGS.master,
                is_chief=(FLAGS.task == 0),
                init_fn=_get_init_fn(),
                summary_op=summary_op,
                number_of_steps=FLAGS.max_number_of_steps,
                log_every_n_steps=FLAGS.log_every_n_steps,
                save_summaries_secs=FLAGS.save_summaries_secs,
                save_interval_secs=FLAGS.save_interval_secs,
                sync_optimizer=optimizer if FLAGS.sync_replicas else None,
                session_config=session_config,
                )

DataLossError (see above for traceback): Unable to open table file

就是save_path书写方式不对！！！
路径不能精确到文件！
如保存操作为saver.save(sess, “train_model”)
那么会产生四个文件，分别为：

  1. checkpoint
  2. train_model-100.index
  3. train_model-100.meta
  4. train_model-100.data-00000-of-00001

那么导入时，saver.restore()中的save_path应填”train_model-100”，不是“train_model-100.data-00000-of-00001”，也不是“train_model”！

------ 本文结束------
坚持原创技术分享，您的支持将鼓励我继续创作！

欢迎关注我的其它发布渠道