1
0
mirror of synced 2026-02-16 18:07:56 +08:00

add: id register

This commit is contained in:
koukouchan
2022-08-24 21:34:09 +08:00
parent 9ab60d7cce
commit 27caa402d4
12 changed files with 423 additions and 211 deletions

View File

@@ -1,17 +0,0 @@
from .Options import IdGeneratorOptions
from .SnowFlakeM1 import SnowFlakeM1
class DefaultIdGenerator():
def SetIdGernerator(self, options:IdGeneratorOptions) :
if options.BaseTime < 100000 :
raise ValueError ("BaseTime error.")
self.SnowFlake= SnowFlakeM1(options)
def NextId(self) -> int:
"""
获取新的UUID
"""
return self.SnowFlake.NextId()

View File

@@ -1,27 +0,0 @@
class IdGeneratorOptions():
def __init__(self, workerId = 0, workerIdBitLength = 6, seqBitLength = 6):
# 雪花计算方法,1-漂移算法|2-传统算法默认1。目前只实现了1。
self.Method = 1
# 基础时间ms单位不能超过当前系统时间
self.BaseTime = 1288834974657
# 机器码,必须由外部设定,最大值 2^WorkerIdBitLength-1
self.WorkerId = workerId
# 机器码位长默认值6取值范围 [1, 15](要求:序列数位长+机器码位长不超过22
self.WorkerIdBitLength = workerIdBitLength
# 序列数位长默认值6取值范围 [3, 21](要求:序列数位长+机器码位长不超过22
self.SeqBitLength = seqBitLength
# 最大序列数(含),设置范围 [MinSeqNumber, 2^SeqBitLength-1]默认值0表示最大序列数取最大值2^SeqBitLength-1]
self.MaxSeqNumber = 0
# 最小序列数默认值5取值范围 [5, MaxSeqNumber]每毫秒的前5个序列数对应编号0-4是保留位其中1-4是时间回拨相应预留位0是手工新值预留位
self.MinSeqNumber = 5
# 最大漂移次数默认2000推荐范围500-10000与计算能力有关
self.TopOverCostCount = 2000

View File

@@ -1,11 +0,0 @@
#!/usr/bin/python
# coding=UTF-8
# 组件编号生成器
class SnowFlake(object):
def __init__(self, options):
self.Options = options
def NextId(self) -> int:
return 0

View File

@@ -1,136 +0,0 @@
#!/usr/bin/python
# coding=UTF-8
from .SnowFlake import SnowFlake
from .Options import IdGeneratorOptions
import threading,time
# 组件编号生成器
class SnowFlakeM1(SnowFlake):
def __init__(self, options:IdGeneratorOptions):
# 1.BaseTime
if options.BaseTime != 0:
self.BaseTime = int(options.BaseTime)
else:
self.BaseTime = 1582136402000
# 2.WorkerIdBitLength
if options.WorkerIdBitLength == 0:
self.WorkerIdBitLength = 6
else:
self.WorkerIdBitLength = int(options.WorkerIdBitLength)
# 3.WorkerId
self.WorkerId = options.WorkerId
# 4.SeqBitLength
if options.SeqBitLength == 0:
self.SeqBitLength = 6
else:
self.SeqBitLength = int(options.SeqBitLength)
# 5.MaxSeqNumber
if options.MaxSeqNumber <= 0:
self.MaxSeqNumber = (1 << self.SeqBitLength) - 1
else:
self.MaxSeqNumber = int(options.MaxSeqNumber)
# 6.MinSeqNumber
self.MinSeqNumber = int(options.MinSeqNumber)
# 7.TopOverCostCount
self.TopOverCostCount = int(options.TopOverCostCount)
# 8.Others
self.__TimestampShift = self.WorkerIdBitLength + self.SeqBitLength
self.__CurrentSeqNumber = self.MinSeqNumber
self.__LastTimeTick:int = 0
self.__TurnBackTimeTick:int = 0
self.__TurnBackIndex:int = 0
self.__IsOverCost = False
self.__OverCostCountInOneTerm:int = 0
self.__IDLock = threading.Lock()
def __NextOverCostId(self) -> int:
CurrentTimeTick = self.__GetCurrentTimeTick()
if CurrentTimeTick > self.__LastTimeTick:
self.__LastTimeTick = CurrentTimeTick
self.__CurrentSeqNumber = self.MinSeqNumber
self.__IsOverCost = False
self.__OverCostCountInOneTerm = 0
return self.__CalcId(self.__LastTimeTick)
if self.__OverCostCountInOneTerm >= self.TopOverCostCount:
self.__LastTimeTick = self.__GetNextTimeTick()
self.__CurrentSeqNumber = self.MinSeqNumber
self.__IsOverCost = False
self.__OverCostCountInOneTerm = 0
return self.__CalcId(self.__LastTimeTick)
if self.__CurrentSeqNumber > self.MaxSeqNumber:
self.__LastTimeTick+=1
self.__CurrentSeqNumber = self.MinSeqNumber
self.__IsOverCost = True
self.__OverCostCountInOneTerm+=1
return self.__CalcId(self.__LastTimeTick)
return self.__CalcId(self.__LastTimeTick)
def __NextNormalId(self) -> int:
CurrentTimeTick = self.__GetCurrentTimeTick()
if CurrentTimeTick < self.__LastTimeTick:
if self.__TurnBackTimeTick < 1:
self.__TurnBackTimeTick = self.__LastTimeTick - 1
self.__TurnBackIndex+=1
# 每毫秒序列数的前5位是预留位0用于手工新值1-4是时间回拨次序
# 支持4次回拨次序避免回拨重叠导致ID重复可无限次回拨次序循环使用
if self.__TurnBackIndex > 4:
self.__TurnBackIndex = 1
return self.__CalcTurnBackId(self.__TurnBackTimeTick)
# 时间追平时_TurnBackTimeTick清零
if self.__TurnBackTimeTick > 0:
self.__TurnBackTimeTick = 0
if CurrentTimeTick > self.__LastTimeTick:
self.__LastTimeTick = CurrentTimeTick
self.__CurrentSeqNumber = self.MinSeqNumber
return self.__CalcId(self.__LastTimeTick)
if self.__CurrentSeqNumber > self.MaxSeqNumber:
self.__LastTimeTick+=1
self.__CurrentSeqNumber = self.MinSeqNumber
self.__IsOverCost = True
self.__OverCostCountInOneTerm = 1
return self.__CalcId(self.__LastTimeTick)
return self.__CalcId(self.__LastTimeTick)
def __CalcId(self,useTimeTick) -> int:
self.__CurrentSeqNumber+=1
return ((useTimeTick<<self.__TimestampShift) + (self.WorkerId<<self.SeqBitLength) + self.__CurrentSeqNumber) % int(1e64)
def __CalcTurnBackId(self,useTimeTick) -> int:
self.__TurnBackTimeTick-=1
return ((useTimeTick<<self.__TimestampShift) + (self.WorkerId<<self.SeqBitLength) + self.__TurnBackIndex) % int(1e64)
def __GetCurrentTimeTick(self) -> int:
return int((time.time_ns() / 1e6) - self.BaseTime)
def __GetNextTimeTick(self) -> int:
TempTimeTicker = self.__GetCurrentTimeTick()
while TempTimeTicker <= self.__LastTimeTick:
# 0.001 = 1 mili sec
time.sleep(0.001)
TempTimeTicker = self.__GetCurrentTimeTick()
return TempTimeTicker
def NextId(self) -> int:
self.__IDLock.acquire()
if self.__IsOverCost:
id = self.__NextOverCostId()
else:
id = self.__NextNormalId()
self.__IDLock.release()
return id

View File

@@ -0,0 +1,22 @@
"""
雪花算法生成器接口声明
"""
# !/usr/bin/python
# coding=UTF-8
# 组件编号生成器
class SnowFlake():
def __init__(self, options):
self.options = options
def next_id(self) -> int:
"""
获取新的UUID
"""
return 0

View File

View File

@@ -0,0 +1,35 @@
"""
雪花算法生成器IdGenerator
"""
from . import options
from . import snowflake_m1
class DefaultIdGenerator:
"""
ID生成器
"""
def __init__(self):
self.snowflake = None
def set_id_generator(self, option: options.IdGeneratorOptions):
"""
设置id生成规则信息
"""
if option.base_time < 100000:
raise ValueError("base time error.")
self.snowflake = snowflake_m1.SnowFlakeM1(option)
def next_id(self) -> int:
"""
获取新的UUID
"""
if self.snowflake is None:
raise ValueError("please set id generator at first.")
return self.snowflake.next_id()

View File

@@ -0,0 +1,40 @@
"""
生成器IdGenerator配置选项
"""
class IdGeneratorOptions:
"""
ID生成器配置
- worker_id 全局唯一id, 区分不同uuid生成器实例
- worker_id_bit_length 生成的uuid中worker_id占用的位数
- seq_bit_length 生成的uuid中序列号占用的位数
"""
def __init__(self, worker_id=0, worker_id_bit_length=6, seq_bit_length=6):
# 雪花计算方法,1-漂移算法|2-传统算法), 默认1。目前只实现了1。
self.method = 1
# 基础时间ms单位, 不能超过当前系统时间
self.base_time = 1288834974657
# 机器码, 必须由外部设定, 最大值 2^worker_id_bit_length-1
self.worker_id = worker_id
# 机器码位长, 默认值6, 取值范围 [1, 15](要求:序列数位长+机器码位长不超过22
self.worker_id_bit_length = worker_id_bit_length
# 序列数位长, 默认值6, 取值范围 [3, 21](要求:序列数位长+机器码位长不超过22
self.seq_bit_length = seq_bit_length
# 最大序列数(含), 设置范围 [max_seq_number, 2^seq_bit_length-1]
# 默认值0, 表示最大序列数取最大值2^seq_bit_length-1]
self.max_seq_number = 0
# 最小序列数(含), 默认值5, 取值范围 [5, max_seq_number], 每毫秒的前5个序列数对应编号0-4是保留位
# 其中1-4是时间回拨相应预留位, 0是手工新值预留位
self.min_seq_number = 5
# 最大漂移次数(含), 默认2000, 推荐范围500-10000与计算能力有关
self.top_over_cost_count = 2000

130
Python/source/idregister.py Normal file
View File

@@ -0,0 +1,130 @@
"""
worker id generator
"""
from threading import Thread
import time
import logging
import redis
class Register:
"""
redis封装
- host 代表redis ip
- port 代表redis端口
- max_worker_id worker_id的最大值, 默认为100
- password redis的密码, 默认为空
"""
def __init__(self, host, port, max_worker_id=100, password=None):
self.redis_impl = redis.StrictRedis(host=host, port=port, db=0, password=password)
self.loop_count = 0
self.max_loop_count = 10
self.worker_id_expire_time = 15
self.max_worker_id = max_worker_id
self.worker_id = -1
self.is_stop = False
def get_lock(self, key):
"""
获取分布式全局锁,并设置过期时间为30秒
"""
if self.redis_impl.setnx(key, 1):
self.redis_impl.expire(key, 30)
return True
if self.redis_impl.ttl(key) < 0:
self.redis_impl.expire(key, 30)
return False
def stop(self):
"""
退出注册器的线程
"""
self.is_stop = True
def get_worker_id(self):
"""
获取全局唯一worker_id, 会创建一个线程给worker id续期
失败返回-1
"""
self.loop_count = 0
def extern_life(my_id):
while 1:
time.sleep(self.worker_id_expire_time / 3)
# 是否关闭了
if self.is_stop:
return
# 更新生命周期
if self.worker_id != my_id:
break
try:
self.redis_impl.expire(
f"IdGen:WorkerId:Value:{my_id}",
self.worker_id_expire_time)
except Exception as exe:
logging.error(exe)
continue
self.worker_id = self.__get_next_worker_id()
if self.worker_id > -1:
Thread(target=extern_life, args=[self.worker_id]).start()
return self.worker_id
def __get_next_worker_id(self):
"""
获取全局唯一worker id内部实现
"""
cur = self.redis_impl.incrby("IdGen:WorkerId:Index", 1)
def can_reset():
try:
reset_value = self.redis_impl.incr("IdGen:WorkerId:Value:Edit")
return reset_value != 1
except Exception as ept:
logging.error(ept)
return False
def end_reset():
try:
self.redis_impl.set("IdGen:WorkerId:Value:Edit", 0)
except Exception as ept:
logging.error(ept)
def is_available(worker_id: int):
try:
rst = self.redis_impl.get(f"IdGen:WorkerId:Value:{worker_id}")
return rst != "Y"
except Exception as ept:
logging.error(ept)
return False
if cur > self.max_worker_id:
if can_reset():
self.redis_impl.set("IdGen:WorkerId:Index", -1)
end_reset()
self.loop_count += 1
if self.loop_count > self.max_loop_count:
self.loop_count = 0
return -1
time.sleep(0.2 * self.loop_count)
return self.__get_next_worker_id()
time.sleep(0.2)
return self.__get_next_worker_id()
if is_available(cur):
self.redis_impl.setex(
f"IdGen:WorkerId:Value:{cur}",
self.worker_id_expire_time,
"Y"
)
self.loop_count = 0
return cur
return self.__get_next_worker_id()

View File

@@ -0,0 +1,150 @@
"""
M1生成器
"""
# !/usr/bin/python
# coding=UTF-8
import threading
import time
from .snowflake import SnowFlake
from .options import IdGeneratorOptions
# 组件编号生成器
class SnowFlakeM1(SnowFlake):
"""
M1规则ID生成器配置
"""
def __init__(self, options: IdGeneratorOptions):
# 1.base_time
self.base_time = 1582136402000
if options.base_time != 0:
self.base_time = int(options.base_time)
# 2.worker_id_bit_length
self.worker_id_bit_length = 6
if options.worker_id_bit_length != 0:
self.worker_id_bit_length = int(options.worker_id_bit_length)
# 3.worker_id
self.worker_id = options.worker_id
# 4.seq_bit_length
self.seq_bit_length = 6
if options.seq_bit_length != 0:
self.seq_bit_length = int(options.seq_bit_length)
# 5.max_seq_number
self.max_seq_number = int(options.max_seq_number)
if options.max_seq_number <= 0:
self.max_seq_number = (1 << self.seq_bit_length) - 1
# 6.min_seq_number
self.min_seq_number = int(options.min_seq_number)
# 7.top_over_cost_count
self.top_over_cost_count = int(options.top_over_cost_count)
# 8.Others
self.__timestamp_shift = self.worker_id_bit_length + self.seq_bit_length
self.__current_seq_number = self.min_seq_number
self.__last_time_tick: int = 0
self.__turn_back_time_tick: int = 0
self.__turn_back_index: int = 0
self.__is_over_cost = False
self.___over_cost_count_in_one_term: int = 0
self.__id_lock = threading.Lock()
def __next_over_cost_id(self) -> int:
current_time_tick = self.__get_current_time_tick()
if current_time_tick > self.__last_time_tick:
self.__last_time_tick = current_time_tick
self.__current_seq_number = self.min_seq_number
self.__is_over_cost = False
self.___over_cost_count_in_one_term = 0
return self.__calc_id(self.__last_time_tick)
if self.___over_cost_count_in_one_term >= self.top_over_cost_count:
self.__last_time_tick = self.__get_next_time_tick()
self.__current_seq_number = self.min_seq_number
self.__is_over_cost = False
self.___over_cost_count_in_one_term = 0
return self.__calc_id(self.__last_time_tick)
if self.__current_seq_number > self.max_seq_number:
self.__last_time_tick += 1
self.__current_seq_number = self.min_seq_number
self.__is_over_cost = True
self.___over_cost_count_in_one_term += 1
return self.__calc_id(self.__last_time_tick)
return self.__calc_id(self.__last_time_tick)
def __next_normal_id(self) -> int:
current_time_tick = self.__get_current_time_tick()
if current_time_tick < self.__last_time_tick:
if self.__turn_back_time_tick < 1:
self.__turn_back_time_tick = self.__last_time_tick - 1
self.__turn_back_index += 1
# 每毫秒序列数的前5位是预留位, 0用于手工新值, 1-4是时间回拨次序
# 支持4次回拨次序避免回拨重叠导致ID重复, 可无限次回拨(次序循环使用)。
if self.__turn_back_index > 4:
self.__turn_back_index = 1
return self.__calc_turn_back_id(self.__turn_back_time_tick)
# 时间追平时, _TurnBackTimeTick清零
self.__turn_back_time_tick = min(self.__turn_back_time_tick, 0)
if current_time_tick > self.__last_time_tick:
self.__last_time_tick = current_time_tick
self.__current_seq_number = self.min_seq_number
return self.__calc_id(self.__last_time_tick)
if self.__current_seq_number > self.max_seq_number:
self.__last_time_tick += 1
self.__current_seq_number = self.min_seq_number
self.__is_over_cost = True
self.___over_cost_count_in_one_term = 1
return self.__calc_id(self.__last_time_tick)
return self.__calc_id(self.__last_time_tick)
def __calc_id(self, use_time_tick) -> int:
self.__current_seq_number += 1
return (
(use_time_tick << self.__timestamp_shift) +
(self.worker_id << self.seq_bit_length) +
self.__current_seq_number
) % int(1e64)
def __calc_turn_back_id(self, use_time_tick) -> int:
self.__turn_back_time_tick -= 1
return (
(use_time_tick << self.__timestamp_shift) +
(self.worker_id << self.seq_bit_length) +
self.__turn_back_index
) % int(1e64)
def __get_current_time_tick(self) -> int:
return int((time.time_ns() / 1e6) - self.base_time)
def __get_next_time_tick(self) -> int:
temp_time_ticker = self.__get_current_time_tick()
while temp_time_ticker <= self.__last_time_tick:
# 0.001 = 1 mili sec
time.sleep(0.001)
temp_time_ticker = self.__get_current_time_tick()
return temp_time_ticker
def next_id(self) -> int:
with self.__id_lock:
if self.__is_over_cost:
nextid = self.__next_over_cost_id()
else:
nextid = self.__next_normal_id()
return nextid