def _read_dict(self, filename):
    """ read filename and generate {word: id} dict """
    with open(filename, 'r') as f:
        lines = f.readlines()
    for line in lines:    # 读词与词频
        word, frequency = line.strip('\r\n').split('\t')
        frequency = int(frequency)
        if frequency < self._num_word_threshold:
            continue
        idx = len(self._word_to_id)  # idx随_word_to_id大小递增
        if word == '<UNK>':   # 特殊处理UNK
            self._unk = idx
        self._word_to_id[word] = idx  # 构建处字典：{词： idx}

def sentence_to_id(self, sentence):
    """ 用词的idx编码每个句子 """
    # 切分句子后的每个词，找它的idx，
    word_ids = [self.word_to_id(cur_word) for cur_word in sentence.split()]
    return word_ids

test_word = '你好 呀 ， 参加 比赛 了 吗'

with open(filename, 'r') as f:
    lines = f.readlines()
for line in lines:    # for each line,
    label, content = line.strip('\r\n').split('\t')
    # convert label and content to sequence of ids
    id_label = self._catego_dict.category_to_id(label)
    id_words = self._vocab_dict.sentence_to_id(content)
    id_words = id_words[0: self._encoded_length]           # cut
    padding_length = self._encoded_length - len(id_words)  # pad
    id_words = id_words + [self._vocab_dict.unk for _ in range(padding_length)]

    self._input.append(id_words)
    self._output.append(id_label)
# convert to numpy array
self._input = np.asarray(self._input, dtype=np.int32)
self._output = np.asarray(self._output, dtype=np.int32)

p = np.random.permutation(len(self._input))
self._input = self._input[p]
self._output = self._output[p]

def next_batch(self, batch_size):
    """
    get next batch data
    :param batch_size:
    :return: the next batch of input and output
    """
    end_indicator = self._indicator + batch_size
    if end_indicator > len(self._input):
        self._random_shuffle()
        self._indicator = 0
        end_indicator = batch_size
    if end_indicator > len(self._input):
        raise Exception("batch size : %d is too large" % batch_size)

    batch_input = self._input[self._indicator: end_indicator]
    batch_ouput = self._output[self._indicator: end_indicator]
    self._indicator = end_indicator
    # return what we require
    return batch_input, batch_ouput

with open(train_file, 'r') as f:
    lines = f.readlines()
print(len(lines))
print(lines[204])

label, content = lines[204].strip('\r\n').split('\t')

word_iter = jieba.cut(content)

word_content = ''         # 保存每一个词
for word in word_iter:     # 对切分结果中每个词作如下操作
    word = word.strip(' ')
    if word != '':
        word_content += word + ' '
out_line = '%s\t%s\n' % (label, word_content.strip(' '))
print(out_line)

def generate_seg_file(input_file, output_seg_file):
    """对input_file内容分词"""
    with open(input_file, 'r') as f:    # 读文件
        lines = f.readlines()
    with open(output_seg_file, 'w') as f:                   # 写的方式打开写文件
        for line in lines:                                   # 对于每一行
            label, content = line.strip('\r\n').split('\t')  # 分出lenbel和content
            word_iter = jieba.cut(content)                   # 对content切分成词
            word_content = ''                                    # 保存每个次
            for word in word_iter:                               # 对切分结果中每个词作如下操作：
                word = word.strip(' ')
                if word != '':
                    word_content += word + ' '
            out_line = '%s\t%s\n' % (label, word_content.strip(' '))  # 输入文件每一行的格式
            
            f.write(out_line)     # 写入文件

# 对三个数据集作同样操作
generate_seg_file(train_file, seg_train_file)
generate_seg_file(val_file, seg_val_file)
generate_seg_file(test_file, seg_test_file)

with open(output_seg_file, 'r') as f:
        lines = f.readlines()
        
word_dict = {}
label, content = lines[204].strip('\r\n').split('\t')
for word in content.split():
        word_dict.setdefault(word, 0)
        word_dict[word] += 1
print(word_dict)

sorted_word_dict = sorted(
    word_dict.items(), key = lambda d:d[1], reverse=True)
print(sorted_word_dict)

def generate_vocab_file(input_seg_file, output_vocab_file):
    """将input_seg_file中做词频统计，输出到out_put_file中"""
    with open(input_seg_file, 'r') as f:
        lines = f.readlines()
    word_dict = {}
    for line in lines:
        label, content = line.strip('\r\n').split('\t')
        for word in content.split():
            word_dict.setdefault(word, 0)
            word_dict[word] += 1
    # [(word, frequency), ..., ()]
    sorted_word_dict = sorted(
        word_dict.items(), key = lambda d:d[1], reverse=True)
    with open(output_vocab_file, 'w') as f:
        f.write('<UNK>\t10000000\n')      # 当在测试集中找不到一个词时，用<UNK> 代替
        for item in sorted_word_dict:
            f.write('%s\t%d\n' % (item[0], item[1]))

# 只对已知的训练集执行此操作
generate_vocab_file(seg_train_file, vocab_file)

def generate_category_dict(input_file, category_file):
    """统计类别"""
    with open(input_file, 'r') as f:
        lines = f.readlines()
    category_dict = {}
    for line in lines:
        label, content = line.strip('\r\n').split('\t')
        category_dict.setdefault(label, 0)
        category_dict[label] += 1
    category_number = len(category_dict)
    with open(category_file, 'w') as f:
        for category in category_dict:
            line = '%s\n' % category
            print('%s\t%d' % (category, category_dict[category]))
            f.write(line)
            
generate_category_dict(train_file, category_file)

from DoubleLinkedList import Node, DoubleLinkedList


class FIFO(object):
    def __init__(self, capacity):
        self.capacity = capacity
        self.size = 0
        self.map = {}   # {key: node} as search table
        self.dlist = DoubleLinkedList(self.capacity)  # as the cache

    def get(self, key):
        if key not in self.map:
            return -1
        else:
            node = self.map.get(key)
            return node.value

    def put(self, key, value):
        if self.capacity == 0:
            return

        if key in self.map:   # if this key exist in map
            node = self.map.get(key)  # get this key
            self.dlist.remove(node)   # remove the node with this key
            node.value = value     # update the value of that node
            self.dlist.append(node)  # add node to the tail
        else:
            if self.size == self.capacity:   # if cache is full
                node = self.dlist.pop()
                del self.map[node.key]   # delete the node at the head
                self.size -= 1
            node = Node(key, value)
            self.dlist.append(node)   # add new node to the tail
            self.map[key] = node
            self.size += 1

    def print(self):
        self.dlist.print()

if __name__ == '__main__':
    cache = FIFO(4)
    cache.put(1, 10)
    cache.put(2, 20)
    cache.put(3, 30)
    cache.print()
    cache.put(1, 12)
    cache.print()
    print("------")
    cache.put(4, 40)
    cache.print()
    cache.put(5, 50)
    cache.print()
    cache.put(1, 11)
    cache.print()
    cache.put(3, 30)
    cache.print()
    print(cache.get(1))
    print(cache.get(2))

class Node:
    def __init__(self, key, val):
        self.key = key
        self.value = val
        self.prev = None
        self.next = None

    def __str__(self):
        val = '{%d: %d}' % (self.key, self.value)
        return val

    def __repr__(self):
        val = '{%d: %d}' % (self.key, self.value)
        return val

class DoubleLinkedList:
    def __init__(self, cap=0xffff):
        self.capacity = cap
        self.head = None
        self.tail = None
        self.size = 0

    # add node from head
    def __add_head(self, node):
        if not self.head:
            self.head = node
            self.tail = node
            self.head.next = None
            self.head.prev = None
        else:
            self.head.prev = node
            node.next = self.head
            self.head = node
            self.head.prev = None
        self.size += 1
        return node

    # append node to the tail
    def __add_tail(self, node):
        if not self.tail:
            self.head = node
            self.tail = node
            self.tail.next = None
            self.tail.prev = None
        else:
            self.tail.next = node
            node.prev = self.tail
            self.tail = node
            self.tail.next = None

            self.size += 1
        return node

    # delete tail
    def __del_tail(self):
        if not self.tail:
            return
        node = self.tail
        if node.prev:
            self.tail = node.prev
            self.tail.next = None
            node.prev = None
        else:
            self.tail = self.head = None
        self.size -= 1
        return node

    # delete head
    def __del_head(self):
        if not self.head:  # if empty list
            return
        node = self.head  # assign self.head to a new node
        if node.next:
            self.head = node.next
            self.head.prev = None
            node.next = None
        else:  # only a head exist
            self.head = self.tail = None
        self.size -= 1
        return node

    # remove node in any position
    def __remove(self, node):
        # if node==None, remove tail by default
        if not node:
            node = self.tail
        if node == self.tail:
            self.__del_tail()
        elif node == self.head:
            self.__del_head()
        else:
            node.prev.next = node.next
            node.next.prev = node.prev
        self.size -= 1
        return node

    # APIs
    # pop node from head
    def pop(self):
        return self.__del_head()

    # add node to the tail
    def append(self, node):
        return self.__add_tail(node)

    # add node to the head
    def append_head(self, node):
        return self.__add_head(node)

    # remove
    def remove(self, node=None):
        return self.__remove(node)

    def print(self):
        p = self.head
        line = ''
        while p:
            line += '%s' % p
            p = p.next
            if p:
                line += '->'
        if not line:
            print("empty double linked list")
        print(line)

def separable_conv_block(x, output_channel_number, name):
    """
    x: input 
    output_channel_number: the output channel of the entire block, 
                            又是1x1卷据层的卷积核个数(卷积核个数==输出通道数)
    name: namespace
    """
    with tf.variable_scope(name):
        # get channel number:
        input_channel = x.get_shape().as_list()[-1]
        
        # split channels to a channel list:
        # channel_wise_x: [channel1, channel2, ...]
        channel_wise_x = tf.split(x, input_channel, axis = 3)
        
        # 对每一个通道分别执行3x3的卷积操作
        output_channels = []
        for i in range(len(channel_wise_x)):      
            output_channel = tf.layers.conv2d(channel_wise_x[i],
                                              1,
                                              (3, 3),
                                              strides = (1,1),
                                              padding = 'same',
                                              activation = tf.nn.relu,
                                              name = 'conv_%d' % i)
            output_channels.append(output_channel)
        
        # concat along channel(index=3)
        concat_layer = tf.concat(output_channels, axis = 3)
        
        # 经过一个1x1的卷积操作
        conv1_1 = tf.layers.conv2d(concat_layer,
                                   output_channel_number,
                                   (1,1),
                                   strides = (1,1),
                                   padding = 'same',
                                   activation = tf.nn.relu,
                                   name = 'conv1_1')
    return conv1_1

def inception_block(x, output_channel_for_each_path, name):
    with tf.variable_scope(name): # avoid name conflict
        conv1_1 = tf.layers.conv2d(x,
                                   output_channel_for_each_path[0],
                                   (1, 1),            # 1x1 卷积核
                                   strides = (1,1),
                                   padding = 'same',
                                   activation = tf.nn.relu,
                                   name = 'conv1_1')
        conv3_3 = tf.layers.conv2d(x,
                                   output_channel_for_each_path[1],
                                   (3, 3),            #3x3 卷积核
                                   strides = (1,1),
                                   padding = 'same',
                                   activation = tf.nn.relu,
                                   name = 'conv3_3')
        conv5_5 = tf.layers.conv2d(x,
                                   output_channel_for_each_path[2],
                                   (5, 5),             # 5x5 卷积核
                                   strides = (1,1),
                                   padding = 'same',
                                   activation = tf.nn.relu,
                                   name = 'conv5_5')
        max_pooling = tf.layers.max_pooling2d(x,
                                              (2, 2),
                                              (2, 2),
                                              name = 'max_pooling')
    
    # max_pooling: output = 1/2 input, so need to add 0; 
    max_pooling_shape = max_pooling.get_shape().as_list()[1:]  # size of the output
    input_shape = x.get_shape().as_list()[1:]                  # size of the input
    
    width_padding = (input_shape[0] - max_pooling_shape[0]) // 2 # pad to width
    height_padding = (input_shape[1] - max_pooling_shape[1]) // 2 #pad to height
    
    
    padded_pooling = tf.pad(max_pooling,
                            [[0, 0],
                             [width_padding, width_padding],
                             [height_padding, height_padding],
                             [0, 0]])
    
    # put together all pieses
    concat_layer = tf.concat(
                            [conv1_1, conv3_3, conv5_5, padded_pooling],
                            axis = 3)
    return concat_layer

def residual_block(x, output_channel):
    """residual connection implementation"""
    input_channel = x.get_shape().as_list()[-1]
    if input_channel * 2 == output_channel: #如果本层的输出通道数是输入通道数的2倍
        increase_dim = True
        strides = (2, 2)
    elif input_channel == output_channel:  #如果本层的输出通道数与输入通道数相同
        increase_dim = False
        strides = (1, 1)
    else:
        raise Exception("input channel can't match output channel")
    conv1 = tf.layers.conv2d(x,
                             output_channel,
                             (3,3),
                             strides = strides,
                             padding = 'same',
                             activation = tf.nn.relu,
                             name = 'conv1')
    conv2 = tf.layers.conv2d(conv1,
                             output_channel,
                             (3, 3),
                             strides = (1, 1),
                             padding = 'same',
                             activation = tf.nn.relu,
                             name = 'conv2')
    # 如果通道数翻倍了
    if increase_dim:
        # [None, image_width, image_height, channel] -> [,,,channel*2]
        pooled_x = tf.layers.average_pooling2d(x,
                                               (2, 2),
                                               (2, 2),
                                               padding = 'valid')
        # 使padded_x 与 conv2 大小相同
        padded_x = tf.pad(pooled_x,
                          [[0,0],
                           [0,0],
                           [0,0],
                           [input_channel // 2, input_channel // 2]])
    # 如果通道数未翻倍
    else:
        padded_x = x  # 全等映射 

    output_x = conv2 + padded_x 
    return output_x