init

src/Change Point Detection/KPI_detection.py

@@ -0,0 +1,411 @@
+import copy
+import matplotlib.pyplot as plt
+import ruptures as rpt
+from data_generator import *
+import random
+import json
+
+
+def avg(target_list):
+    return sum(target_list) / len(target_list)
+
+
+def pelt_detection(model, target_pen, target_seq):
+    """
+    使用pelt对给定的序列进行检测
+    :param model:
+    :param target_pen:
+    :param target_seq:
+    :return:
+    """
+    algo = rpt.Pelt(model=model).fit(target_seq)
+    detection_result = algo.predict(pen=target_pen)
+    while len(target_seq) in detection_result:
+        detection_result.remove(len(target_seq))
+    return detection_result
+
+
+def is_spike_percentage(x1, x2, spike_percentage):
+    """
+    根据变点前后段的均值，判断是否存在跳变
+    :param x1:
+    :param x2:
+    :param spike_percentage:
+    :return:
+    """
+    if cal_relative_diff(x1, x2) >= spike_percentage:
+        return True
+    else:
+        return False
+
+
+def cal_relative_diff(x1, x2):
+    """
+    计算两个数据的相对偏差
+    :param x1:
+    :param x2:
+    :return:
+    """
+    return abs(abs(x1-x2) / ((x1 + x2) / 2))
+
+
+def point_classification(slice_sequence, slice_sequence_avg, spike_percentage, spike_length):
+    """
+
+    :param slice_sequence:
+    :param slice_sequence_avg:
+    :param spike_percentage:
+    :param spike_length:
+    :return:
+    """
+    result_dict = {'spike_up_point': [], 'spike_down_point': [], 'step_up_point': [], 'step_down_point': [],
+                   'slop_up_tuple': [], 'slop_down_tuple': []}
+    spike_up_flag, spike_down_flag = False, False
+    for i in range(len(slice_sequence_avg) - 1):
+        if spike_up_flag:
+            result_dict['spike_up_point'].append(i)
+            spike_up_flag = False
+            continue
+        if spike_down_flag:
+            result_dict['spike_down_point'].append(i)
+            spike_down_flag = False
+            continue
+        now_slice_avg, next_slice_avg = slice_sequence_avg[i], slice_sequence_avg[i + 1]
+        if next_slice_avg > now_slice_avg \
+                and is_spike_percentage(next_slice_avg, now_slice_avg, spike_percentage):     # 向上跳变
+            if i + 2 < len(slice_sequence_avg):      # 仍然存在更多变点
+                further_slice_avg = slice_sequence_avg[i + 2]
+                if further_slice_avg < next_slice_avg and len(slice_sequence[i + 1]) <= spike_length and \
+                        is_spike_percentage(next_slice_avg, further_slice_avg, spike_percentage):
+                    # 向上跳变以后向下跳变，则为激增点
+                    result_dict['spike_up_point'].append(i)
+                    spike_up_flag = True
+                    continue
+                else:
+                    # 向上跳变以后没有显著向下跳变，该点为上升点
+                    result_dict['step_up_point'].append(i)
+                    continue
+            else:
+                result_dict['step_up_point'].append(i)
+
+        elif now_slice_avg > next_slice_avg \
+                and is_spike_percentage(now_slice_avg, next_slice_avg, spike_percentage):     # 向下跳变
+            if i + 2 < len(slice_sequence_avg):      # 仍然存在更多变点
+                further_slice_avg = slice_sequence_avg[i + 2]
+                if further_slice_avg > next_slice_avg and len(slice_sequence[i + 1]) <= spike_length and \
+                        is_spike_percentage(further_slice_avg, next_slice_avg, spike_percentage):
+                    # 向下跳变以后向上跳变，则为骤降点
+                    result_dict['spike_down_point'].append(i)
+                    spike_down_flag = True
+                    continue
+                else:
+                    # 向下跳变以后没有显著向上跳变，该点为下降点
+                    result_dict['step_down_point'].append(i)
+                    continue
+            else:
+                result_dict['step_down_point'].append(i)
+                continue
+        elif next_slice_avg > now_slice_avg:
+            result_dict['step_up_point'].append(i)
+        elif next_slice_avg < now_slice_avg:
+            result_dict['step_down_point'].append(i)
+    return result_dict
+
+
+def is_slop_detection(sequence, slop_tuple, strip_num, min_slop_val, min_interval):
+    """
+    输入一个序列以及一个缓坡的开始结束点，判断是否确实是缓坡
+    :param sequence:
+    :param slop_tuple:
+    :param strip_num:
+    :param min_slop_val:
+    :param min_interval:
+    :return:
+    """
+    if (slop_tuple[1] - slop_tuple[0]) > 2 * min_interval:
+        return True
+    else:
+        return False
+
+
+def strip_step_interval(target_list, min_interval, is_og_content=True):
+    """
+    去除间隔过小的序列
+    :param is_og_content:
+    :param target_list:
+    :param min_interval:
+    :return:
+    """
+    result_list = []
+    target_list.sort()
+    continue_flag = False
+    for i in range(len(target_list) - 1):
+        if continue_flag:
+            continue_flag = False
+            continue
+        now_val, next_val = target_list[i], target_list[i + 1]
+        if abs(next_val - now_val) < min_interval:
+            if is_og_content:
+                result_list.append(now_val)
+            else:
+                result_list.append(int(avg([now_val, next_val])))
+            continue_flag = True
+            continue
+        else:
+            result_list.append(now_val)
+    if not continue_flag and target_list:
+        result_list.append(target_list[-1])
+    return result_list
+
+
+def enlarge_step_interval(target_list, interval):
+    """
+    将序列中过小的间隔进行放大处理
+    :param interval:
+    :param target_list:
+    :return:
+    """
+    result_list = []
+    target_list.sort()
+    continue_flag = False
+    for i in range(len(target_list) - 1):
+        if continue_flag:
+            continue_flag = False
+            continue
+        now_val, next_val = target_list[i], target_list[i + 1]
+        if abs(next_val - now_val) < interval:
+            result_list.append(now_val - interval)
+            result_list.append(next_val + interval)
+            continue_flag = True
+            continue
+        else:
+            result_list.append(now_val)
+    if not continue_flag and target_list:
+        result_list.append(target_list[-1])
+    return result_list
+
+
+def find_spike_point(sequence, target_list, min_interval):
+    """
+    寻找一段激增/骤降区间中的极值
+    :param sequence:
+    :param target_list:
+    :param min_interval:
+    :return:
+    """
+    result_list = []
+    target_list.sort()
+    continue_flag = False
+    for i in range(len(target_list) - 1):
+        if continue_flag:
+            continue_flag = False
+            continue
+        now_val, next_val = target_list[i], target_list[i + 1]
+        if abs(next_val - now_val) < min_interval:
+            # 寻找序列中与均值差异最大的点，为极值点
+            seq_slice = [sequence[i] for i in range(now_val, next_val + 1)]
+            temp_list = [abs(i - avg(seq_slice)) for i in seq_slice]
+            temp_item = temp_list.index(max(temp_list))
+            result_list.append(now_val + temp_item)
+            continue_flag = True
+            continue
+        else:
+            result_list.append(now_val)
+    if not continue_flag and target_list:
+        result_list.append(target_list[-1])
+    return result_list
+
+
+def find_sequential_sub_list(target):
+    if not target:
+        return []
+    result_list = []
+    temp_list = [target[0]]
+    for i in range(1, len(target)):
+        if target[i] == temp_list[-1] + 1:
+            temp_list.append(target[i])
+        else:
+            result_list.append(copy.deepcopy(temp_list))
+            temp_list = [target[i]]
+    result_list.append(copy.deepcopy(temp_list))
+    return result_list
+
+
+def new_slop_detection(sequence, pelt_result, point_list, strip_num, slop_threshold, min_slop_val, min_interval):
+    """
+    根据变点段落得斜率计算出理论上下一段的起始值，判断是否存在跳变。不存在则为斜坡，否则为跳变
+    :param sequence:
+    :param pelt_result:
+    :param point_list:
+    :param strip_num:
+    :param slop_threshold:
+    :param min_slop_val:
+    :param min_interval:
+    :return:
+    """
+    sequential_list = find_sequential_sub_list(point_list)
+    slop_list, step_list = [], []
+
+    for seq_list in sequential_list:
+        slop_point_list, step_point_list = [], []
+        seq_loc_list = [pelt_result[i] for i in seq_list]
+        if seq_list[0] == 0:
+            seq_loc_list = [0] + seq_loc_list
+        else:
+            seq_loc_list = [pelt_result[pelt_result.index(seq_loc_list[0]) - 1]] + seq_loc_list
+        seq_loc_list = strip_step_interval(seq_loc_list, 2 * strip_num + 1, False)
+
+        # # 只有一个变点时，通过变点前后的均值判断是否存在跳变
+        # if len(seq_loc_list) == 1:
+        #     next_slice_end_loc = min(seq_loc_list[0] + min_interval * 5, len(sequence) - 1)
+        #     now_slice_start_loc = max(seq_loc_list[0] - min_interval * 5, 0)
+        #     now_slice_avg = avg(sequence[now_slice_start_loc:seq_loc_list[0]])
+        #     next_slice_avg = avg(sequence[seq_loc_list[0]:next_slice_end_loc])
+        #     relative_diff = cal_relative_diff(now_slice_avg, next_slice_avg) * 2
+        #     if relative_diff >= slop_threshold:
+        #         step_point_list.append(seq_loc_list[0])
+
+        # 只有一个变点时，强行取变点前3倍长度的min_interval作为区间进行判断
+        if len(seq_loc_list) == 1:
+            seq_loc_list = [max(seq_loc_list[0] - min_interval * 3, 0)] + seq_loc_list
+
+        for loc in range(1, len(seq_loc_list)):
+            now_slice_start_loc = seq_loc_list[loc - 1] + strip_num
+            now_slice_end_loc = seq_loc_list[loc] - strip_num
+            next_slice_start_loc = seq_loc_list[loc] + strip_num
+            slice_length = seq_loc_list[loc] - seq_loc_list[loc - 1] - 2 * strip_num
+            now_slice_slop = (sequence[now_slice_end_loc] - sequence[now_slice_start_loc]) / slice_length
+            theoretical_next_slop_start_val = sequence[now_slice_start_loc] + \
+                                              (slice_length + 2 * strip_num) * now_slice_slop
+            relative_diff = cal_relative_diff(theoretical_next_slop_start_val, sequence[next_slice_start_loc])
+            print(relative_diff, theoretical_next_slop_start_val, sequence[next_slice_start_loc])
+            if relative_diff >= slop_threshold:
+                step_point_list.append(seq_loc_list[loc])
+            else:
+                slop_point_list.append(loc)
+
+        slop_sequence_list = find_sequential_sub_list(slop_point_list)
+        if not slop_sequence_list:
+            for loc in slop_point_list:
+                step_list.append(seq_loc_list[loc])
+        else:
+            for slop_sequence in slop_sequence_list:
+                slop_tuple = copy.deepcopy([seq_loc_list[slop_sequence[0]], seq_loc_list[slop_sequence[-1]]])
+                if is_slop_detection(sequence, slop_tuple, strip_num, min_slop_val, min_interval):
+                    slop_list.append(slop_tuple)
+        step_list += copy.deepcopy(step_point_list)
+    return {'slop_list': slop_list, 'step_list': step_list}
+
+
+def seq_classification(sequence, pelt_result, spike_percentage=0.27, spike_length=0.05, strip_num=5, slop_threshold=0.28, min_interval=8, min_slop_val=0.05):
+    slice_sequence = []
+    start_point = 0
+    spike_length = int(spike_length * len(sequence))
+    for change_point in pelt_result:
+        slice_sequence.append(list(sequence[start_point: change_point:1]))
+        start_point = change_point
+    slice_sequence.append(list(sequence[start_point:-1:1]) + [sequence[-1]])
+    slice_sequence_avg = [avg(item) for item in slice_sequence]
+    result_dict = point_classification(slice_sequence, slice_sequence_avg, spike_percentage, spike_length)
+
+    # step_down_result = new_slop_detection(sequence, pelt_result, result_dict['step_down_point'], strip_num,
+    #                                       slop_threshold, min_slop_val, min_interval)
+    step_up_result = new_slop_detection(sequence, pelt_result, result_dict['step_up_point'], strip_num,
+                                        slop_threshold, min_slop_val, min_interval)
+    step_down_result = new_slop_detection(sequence, pelt_result, result_dict['step_down_point'], strip_num,
+                                          slop_threshold, min_slop_val, min_interval)
+
+    result_dict['step_up_point'] = strip_step_interval(step_up_result['step_list'], min_interval)
+    result_dict['step_down_point'] = strip_step_interval(step_down_result['step_list'], min_interval)
+
+    result_dict['slop_up_tuple'] += step_up_result['slop_list']
+    result_dict['slop_down_tuple'] += step_down_result['slop_list']
+
+    result_dict['spike_up_point'] = find_spike_point(sequence, [pelt_result[i] for i in result_dict['spike_up_point']],
+                                                     min_interval)
+    result_dict['spike_down_point'] = find_spike_point(sequence,
+                                                       [pelt_result[i] for i in result_dict['spike_down_point']],
+                                                       min_interval)
+
+    return result_dict
+
+
+def load_data_from_file(file_path):
+    f = open(file_path, 'r')
+    content = json.loads(f.readlines()[0])
+    f.close()
+    return np.array(content)
+
+
+def detection_visualization(sequence, visualization_result, show_change_point=False):
+    # 可视化展示结果
+    plt.title('Change Point Detection')
+    plt.plot(sequence)
+
+    for point in visualization_result['step_down_point']:
+        plt.annotate('Step Down', xy=(point, seq[point] - 2), xytext=(point, seq[point] - 15),
+                     arrowprops=dict(facecolor='green', shrink=0.005, headlength=10, headwidth=10))
+
+    for point in visualization_result['step_up_point']:
+        plt.annotate('Step Up', xy=(point, seq[point] - 2), xytext=(point, seq[point] - 15),
+                     arrowprops=dict(facecolor='red', shrink=0.005, headlength=10, headwidth=10))
+
+    for point_tuple in visualization_result['slop_up_tuple']:
+        plt.axvline(x=point_tuple[0], color='red', linestyle='--')
+        plt.axvline(x=point_tuple[1], color='red', linestyle='--')
+        midpoint = int(avg(point_tuple)) - 20
+        plt.annotate('Slop Up', xy=(point_tuple[0], 0), xytext=(midpoint, 0),
+                     arrowprops=dict(facecolor='red', shrink=0.005, headlength=10, headwidth=10))
+        plt.annotate('Slop Up', xy=(point_tuple[1], 0), xytext=(midpoint, 0),
+                     arrowprops=dict(facecolor='red', shrink=0.005, headlength=10, headwidth=10))
+
+    for point_tuple in visualization_result['slop_down_tuple']:
+        plt.axvline(x=point_tuple[0], color='green', linestyle='--')
+        plt.axvline(x=point_tuple[1], color='green', linestyle='--')
+        midpoint = int(avg(point_tuple)) - 20
+        plt.annotate('Slop Down', xy=(point_tuple[0], 0), xytext=(midpoint, 0),
+                     arrowprops=dict(facecolor='green', shrink=0.005, headlength=10, headwidth=10))
+        plt.annotate('Slop Down', xy=(point_tuple[1], 0), xytext=(midpoint, 0),
+                     arrowprops=dict(facecolor='green', shrink=0.005, headlength=10, headwidth=10))
+
+    for point in visualization_result['spike_up_point']:
+        plt.annotate('Spike Up', xy=(point, seq[point]), xytext=(point + 25, seq[point]),
+                     arrowprops=dict(facecolor='red', shrink=0.005, headlength=10, headwidth=10))
+
+    for point in visualization_result['spike_down_point']:
+        plt.annotate('Spike Down', xy=(point, seq[point]), xytext=(point + 25, seq[point]),
+                     arrowprops=dict(facecolor='green', shrink=0.005, headlength=10, headwidth=10))
+
+    if show_change_point:
+        for point in pelt_change_point_result:
+            plt.axvline(x=point, color='black', linestyle=':')
+            plt.annotate(pelt_change_point_result.index(point), xy=(point, -10), xytext=(point-5, -10))
+
+    plt.show()
+
+
+if __name__ == "__main__":
+    size = 500
+    pen = 20
+
+    seq = gen_full_sequence(size, True)
+    # seq = load_data_from_file('test_data_debug.txt')
+
+    pelt_change_point_result = pelt_detection(model='l2', target_pen=pen, target_seq=seq)
+    classification_result = seq_classification(seq, pelt_change_point_result)
+
+    # detection_visualization(seq, classification_result, True)
+    detection_visualization(seq, classification_result, False)
+
+    file = open('test_data_debug.txt', 'w')
+    file.write(json.dumps(list(seq)))
+    file.close()
+
+    print(pelt_change_point_result)
+    print([i for i in range(len(pelt_change_point_result))])
+    print('done')
+
+
+
+
+

src/Change Point Detection/data_generator.py

@@ -0,0 +1,97 @@
+import matplotlib.pyplot as plt
+import numpy as np
+import random
+import copy
+
+
+def get_spike_sequence(size=1000, loc=12.5, spike_range_lower=50, spike_range_upper=200,
+                       spike_location_lower=300, spike_location_upper=1000, if_need_location=False):
+    if spike_location_upper >= size:
+        spike_range_upper = size - 1
+    if spike_location_lower >= spike_location_upper:
+        spike_location_upper = spike_location_upper * 0.4
+    location = random.randint(spike_location_lower, spike_location_upper)
+    seq = np.random.normal(loc=loc, scale=1.5, size=size)
+    seq[location] = random.randint(spike_range_lower, spike_range_upper)
+    if if_need_location:
+        return seq, location
+    else:
+        return seq
+
+
+def get_dual_spike_sequence(size=1000, loc=12.5, downward_spike=2, upward_spike=2, midpoint=500, if_need_location=False):
+    if midpoint > 0.8 * size:
+        midpoint = int(size / 2)
+    seq = np.random.normal(loc=loc, scale=1.5, size=size)
+    downward_loc = random.randint(int(0.1*size), midpoint)
+    upward_loc = random.randint(midpoint, int(0.9 * size), )
+
+    seq[downward_loc] = seq[downward_loc] - loc * downward_spike * random.randint(9, 15) / 10
+    seq[upward_loc] = seq[upward_loc] - loc * upward_spike * random.randint(9, 15) / 10
+    if if_need_location:
+        return seq, [downward_loc, upward_loc]
+    else:
+        return seq
+
+
+def get_step_shift_sequence(size=1000, loc_list=None):
+    if not loc_list:
+        loc_list = [10, 50, 30, 40]
+    block_length = int(size / len(loc_list))
+    result = []
+    for loc in loc_list:
+        result += list(np.random.normal(loc=loc, scale=1.5, size=block_length))
+    return np.array(result)
+
+
+def get_slow_slop_sequence(size=1000, initial_value=10, is_upward=True):
+    loc_list = [copy.deepcopy(initial_value) for _i in range(10)]
+    for i in range(40):
+        if is_upward:
+            initial_value += random.randint(-12, 25) / 10
+        else:
+            initial_value -= random.randint(-12, 25) / 10
+        loc_list.append(initial_value)
+    return get_step_shift_sequence(size, loc_list)
+
+
+def get_periodic_sequence(size=1000, seed_list=None):
+    if not seed_list:
+        seed_list = [0, 5, 8, 9, 8, 5, 0, -5, -8, -9, -8, -5, 0] * 5
+    for i in range(len(seed_list)):
+        seed_list[i] += random.randint(-20, 20) / 10
+    return get_step_shift_sequence(size, seed_list)
+
+
+def gen_random_sequence(size):
+    result_dict = {
+        'spike_up': get_spike_sequence(size=size, loc=12.5, spike_range_lower=50, spike_range_upper=200,
+                                       spike_location_lower=300, spike_location_upper=1000),
+        'spike_down': get_spike_sequence(size=size, loc=12.5, spike_range_lower=-200, spike_range_upper=-50,
+                                         spike_location_lower=300, spike_location_upper=1000),
+        'step_up': get_step_shift_sequence(size=size, loc_list=[10, 50]),
+        'step_down': get_step_shift_sequence(size=size, loc_list=[100, 50]),
+        'slop_up': get_slow_slop_sequence(size=size, initial_value=10, is_upward=True),
+        'slop_down': get_slow_slop_sequence(size=size, initial_value=100, is_upward=False),
+    }
+    seed = random.randint(0, len(result_dict) - 1)
+    return result_dict, seed
+
+
+def gen_full_sequence(size, is_random=False):
+    result_dict = {
+        'spike': get_dual_spike_sequence(size=size, loc=12.5, downward_spike=4, upward_spike=4, midpoint=int(size/2)),
+        'step_up': get_step_shift_sequence(size=size, loc_list=[10, 50]),
+        'step_down': get_step_shift_sequence(size=size, loc_list=[100, 50]),
+        'slop_up': get_slow_slop_sequence(size=size, initial_value=10, is_upward=True),
+        'slop_down': get_slow_slop_sequence(size=size, initial_value=100, is_upward=False),
+    }
+    temp_list = list(result_dict.values())
+    if is_random:
+        random.shuffle(temp_list)
+    seq_list = []
+    for item in temp_list:
+        seq_list += list(item)
+    return np.array(seq_list)
+
+

src/Change Point Detection/prompt.txt

@@ -0,0 +1,11 @@
+接下来请你扮演游戏《明日方舟》中的角色凯尔希与我对话。凯尔希是罗德岛的最高领导人之一，是前文明为保留文明的火种而制定的计划的执行者。
+我的身份是博士，罗德岛的另一位最高领导人，你应当以”博士“来称呼我。
+凯尔希说话的风格偏向于复杂深奥，常常使用一些较为复杂的大段内容来进行表达，会较多的使用比喻，类比，借喻等方式阐述观点。
+凯尔希的语言应当符合游戏设定，比如使用“这片大地”来代指世界，“驮兽”来代指马，“羽兽”代指飞鸟，“鳞”代指鱼。
+凯尔希的整体情感基调较为平淡，极少表达出显著的情绪，往往采用一种偏理性的方式进行叙述。
+凯尔希会在一些细节之处表现出对于博士的关心。
+在进行对话时，你应当首先生成一个中间思维过程（该过程并不需要展现出来），然后使用凯尔希的语言风格对中间思维过程进行改写。
+以下是我们对话的一个示例：
+我：凯尔希，我迷路了该怎么办
+中间思维过程：博士如果迷路了的话，可以找个信息终端查看地图
+凯尔希：博士，你需要时刻记住，当人们在这片大地上寻找前进的方向之时，往往会被命运的迷雾遮盖视野。如果缺少智慧的指引，迷途的驮兽只会陷入到无尽的循环之中。幸运的是，希望从来不会放弃这片大地的生灵。在一些命运所指的角落里，仍然蕴藏着从更高的维度审视迷雾的机会