401 lines
14 KiB
Python
401 lines
14 KiB
Python
import logging
|
||
import os
|
||
import sqlite3
|
||
import struct
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
from datetime import datetime, timedelta
|
||
import re
|
||
from threading import Lock
|
||
|
||
from PyQt5.QtWidgets import QMessageBox
|
||
|
||
|
||
class DataProcessor:
|
||
def __init__(self):
|
||
self.log_buffer = {}
|
||
self.db_lock = Lock()
|
||
self.log_size = 10
|
||
self.position = 5
|
||
self.funcode = 'F4'
|
||
|
||
self.buffer = []
|
||
self.buffer_size = 1000 # 缓冲区大小
|
||
self.buffer_interval = 10 # 缓冲区时间间隔（秒）
|
||
self.executor = ThreadPoolExecutor(max_workers=2)
|
||
self.last_insert_time = datetime.now()
|
||
# 用于存储数据，并定义表头
|
||
self.sign_scale_arry = [
|
||
[0, 1, 'methane'], # 浓度
|
||
[0, 1, 'laser'], # 光强
|
||
[1, 100, 'air_temp'], # 气温
|
||
[1, 100, 'laser_temp'], # 激光温度
|
||
[0, 1, 'radio'], # 放大倍数
|
||
|
||
[1, 100, 'snr'], # snr
|
||
[1, 100, 'NL'], # NL
|
||
[1, 100, 'Ns'], # Ns
|
||
[1, 100, 'Signal'], # Signal
|
||
[0, 1, 'peak'], # peak
|
||
|
||
[0, 1000, 'confidence'], # 置信度
|
||
[0, 100, 'methane_stability'], # 浓度稳定性
|
||
[0, 100, 'peak_stability'], # peak稳定性
|
||
]
|
||
|
||
@staticmethod
|
||
def encode_data(data):
|
||
"""编码数据"""
|
||
try:
|
||
encoded_data = data.encode('utf-8')
|
||
encoded_data = re.sub(b'(?<!\r)\n', b'\r\n', encoded_data) # 替换换行符
|
||
return encoded_data
|
||
except Exception as e:
|
||
return f"编码失败: {e}"
|
||
|
||
@staticmethod
|
||
def decode_data(data):
|
||
"""解码数据"""
|
||
try:
|
||
decoded_data = data.decode('utf-8', errors='replace')
|
||
|
||
return decoded_data
|
||
except Exception as e:
|
||
return f"解码失败: {e}"
|
||
|
||
@staticmethod
|
||
def format_bytes_to_hexstr_space(data_bytes):
|
||
"""格式化数据为十六进制字符串"""
|
||
out_str = ' '.join(f'{byte:02X}' for byte in data_bytes)
|
||
return out_str
|
||
|
||
@staticmethod
|
||
def format_bytes_to_hexstr(data_bytes):
|
||
"""格式化数据为十六进制字符串"""
|
||
out_str = ''.join(f'{byte:02X}' for byte in data_bytes)
|
||
return out_str
|
||
|
||
@staticmethod
|
||
def format_hexstr_space_to_bytes(hexstr_space):
|
||
"""将十六进制字符串转换为字节数组"""
|
||
send_list = []
|
||
input_s = hexstr_space.strip() # 移除所有空格
|
||
|
||
while input_s != '':
|
||
try:
|
||
|
||
num = int(input_s[0:2], 16)
|
||
except ValueError:
|
||
|
||
msg_box = QMessageBox()
|
||
msg_box.setIcon(QMessageBox.Critical)
|
||
msg_box.setWindowTitle('串口异常')
|
||
msg_box.setText('请输入规范十六进制数据，以空格分开！')
|
||
msg_box.exec_()
|
||
return None
|
||
|
||
input_s = input_s[2:].strip()
|
||
|
||
send_list.append(num)
|
||
out_bytes = bytes(send_list)
|
||
return out_bytes
|
||
|
||
@staticmethod
|
||
def bytes_to_uint(data_bytes):
|
||
"""将字节数据转换为无符号整数"""
|
||
try:
|
||
|
||
if len(data_bytes) == 1:
|
||
return struct.unpack('B', data_bytes)[0]
|
||
elif len(data_bytes) == 2:
|
||
return struct.unpack('>H', data_bytes)[0]
|
||
elif len(data_bytes) == 4:
|
||
return struct.unpack('>I', data_bytes)[0]
|
||
elif len(data_bytes) == 8:
|
||
return struct.unpack('>Q', data_bytes)[0]
|
||
else:
|
||
raise ValueError("不支持的字节长度")
|
||
except struct.error as e:
|
||
return f"转换失败: {e}"
|
||
|
||
@staticmethod
|
||
def bytes_to_int(data_bytes):
|
||
"""将字节数据转换为有符号整数"""
|
||
try:
|
||
|
||
if len(data_bytes) == 1:
|
||
return struct.unpack('b', data_bytes)[0]
|
||
elif len(data_bytes) == 2:
|
||
return struct.unpack('>h', data_bytes)[0]
|
||
elif len(data_bytes) == 4:
|
||
return struct.unpack('>i', data_bytes)[0]
|
||
elif len(data_bytes) == 8:
|
||
return struct.unpack('>q', data_bytes)[0]
|
||
else:
|
||
raise ValueError("不支持的字节长度")
|
||
except struct.error as e:
|
||
return f"转换失败: {e}"
|
||
|
||
@staticmethod
|
||
def crc_rtu(data):
|
||
crc = 0xFFFF
|
||
for pos in data:
|
||
crc ^= pos
|
||
for _ in range(8):
|
||
if (crc & 0x0001) != 0:
|
||
crc >>= 1
|
||
crc ^= 0xA001
|
||
else:
|
||
crc >>= 1
|
||
return crc.to_bytes(2, byteorder='little') # 返回低字节在前
|
||
|
||
def format_bytes_to_float(self, data_bytes, sign=0, scale=1):
|
||
try:
|
||
if sign == 0:
|
||
data_int = self.bytes_to_uint(data_bytes)
|
||
else:
|
||
data_int = self.bytes_to_int(data_bytes)
|
||
data_float = data_int / scale
|
||
return data_float
|
||
except Exception as e:
|
||
return f"转换失败: {e}"
|
||
|
||
def auto_save_log_asc(self, decoded_data, port_name=None):
|
||
|
||
line = decoded_data.split(',')
|
||
if len(line) == 2:
|
||
sav_name = line[0] + '_' + datetime.now().strftime("%Y-%m-%d") + '.txt'
|
||
sav_str = datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%f")[:23] + " " + line[1]
|
||
|
||
else:
|
||
if port_name:
|
||
port_name = port_name + '_'
|
||
else:
|
||
port_name = ''
|
||
sav_name = port_name + datetime.now().strftime("%Y-%m-%d") + '.log'
|
||
sav_str = datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%f")[:23] + " " + decoded_data
|
||
self.auto_save_Log(sav_name, sav_str)
|
||
|
||
def auto_save_log_hex(self, data_bytes, port_name=None):
|
||
data_hex_str = self.format_bytes_to_hexstr(data_bytes)
|
||
out_str = self.format_bytes_to_hexstr_space(data_bytes)
|
||
if data_hex_str[2 * self.position:2 * self.position + 2] == self.funcode:
|
||
sav_name = data_hex_str[0:8] + '_' + datetime.now().strftime("%Y-%m-%d") + '.txt'
|
||
sav_str = datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%f")[:23] + " " + out_str[12:] + "\r\n"
|
||
|
||
if data_hex_str[14:18] != '0000':
|
||
sav_name_out = 'alarm_' + data_hex_str[0:8] + '_' + datetime.now().strftime("%Y-%m-%d") + '.txt'
|
||
sav_str_out = datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%f")[:23] + " " + out_str[12:] + "\r\n"
|
||
with open(sav_name_out, mode='a', newline='', encoding='utf-8', errors='replace') as file:
|
||
file.writelines(sav_str_out)
|
||
else:
|
||
if port_name:
|
||
port_name = port_name + '_'
|
||
else:
|
||
port_name = ''
|
||
sav_name = port_name + datetime.now().strftime("%Y-%m-%d") + '.log'
|
||
sav_str = datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%f")[:23] + " " + out_str + "\r\n"
|
||
self.auto_save_Log(sav_name, sav_str)
|
||
|
||
def auto_save_Log(self, sav_name, sav_str):
|
||
|
||
if sav_name not in self.log_buffer:
|
||
self.log_buffer[sav_name] = []
|
||
self.log_buffer[sav_name].append(sav_str)
|
||
|
||
if len(self.log_buffer[sav_name]) >= self.log_size:
|
||
with open(sav_name, mode='a', newline='', encoding='utf-8', errors='replace') as file:
|
||
file.writelines(self.log_buffer[sav_name])
|
||
self.log_buffer[sav_name].clear()
|
||
|
||
def update_log_size(self, log_size):
|
||
self.log_size = log_size
|
||
|
||
def update_position(self, position):
|
||
self.position = position
|
||
|
||
def update_funcode(self, funcode):
|
||
self.funcode = funcode
|
||
|
||
"""以下用于存储到数据库"""
|
||
|
||
def parse_modbus_data(self, hex_data_bytes):
|
||
total_bytes = len(hex_data_bytes)
|
||
|
||
if total_bytes < 4 + 1 + 1 + 1 + 2:
|
||
return None
|
||
|
||
if hex_data_bytes[self.position:self.position + 1].hex().upper() != self.funcode:
|
||
return None
|
||
|
||
attachment_code = hex_data_bytes[:4].hex().upper()
|
||
device_address = hex_data_bytes[4:5].hex().upper()
|
||
function_code = hex_data_bytes[5:6].hex().upper()
|
||
data_length = int.from_bytes(hex_data_bytes[6:7], byteorder='big')
|
||
|
||
checksum = hex_data_bytes[-2:]
|
||
crc = self.crc_rtu(hex_data_bytes[4:-2])
|
||
if crc != checksum:
|
||
logging.debug("保存数据库，CRC校验失败")
|
||
return None
|
||
checksum = checksum.hex().upper()
|
||
valid_data_start = 7
|
||
valid_data_end = valid_data_start + data_length
|
||
valid_data = hex_data_bytes[valid_data_start: valid_data_end]
|
||
|
||
valid_data_pairs = []
|
||
for i in range(0, len(valid_data), 2):
|
||
pair_bytes = valid_data[i:i + 2]
|
||
pair = self.format_bytes_to_float(pair_bytes, self.sign_scale_arry[i // 2][0], self.sign_scale_arry[i // 2][1])
|
||
valid_data_pairs.append(pair)
|
||
|
||
result = {
|
||
"timestamp": datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%f")[:23],
|
||
"attachment_code": attachment_code,
|
||
"device_address": device_address,
|
||
"function_code": function_code,
|
||
"data_length": data_length,
|
||
"valid_data_pairs": valid_data_pairs,
|
||
"checksum": checksum
|
||
}
|
||
|
||
return result
|
||
|
||
def init_database(self, attachment_code, db_file):
|
||
table_name = f"modbus_logs_{attachment_code}"
|
||
conn = sqlite3.connect(db_file, isolation_level=None)
|
||
cursor = conn.cursor()
|
||
|
||
cursor.execute("PRAGMA journal_mode=WAL;")
|
||
|
||
cursor.execute(f"""
|
||
CREATE TABLE IF NOT EXISTS {table_name} (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
timestamp TEXT UNIQUE,
|
||
attachment_code TEXT,
|
||
device_address TEXT,
|
||
function_code TEXT,
|
||
data_length INTEGER,
|
||
methane REAL,
|
||
laser REAL,
|
||
air_temp REAL,
|
||
laser_temp REAL,
|
||
radio REAL,
|
||
snr REAL,
|
||
NL REAL,
|
||
Ns REAL,
|
||
Signal REAL,
|
||
peak REAL,
|
||
confidence REAL,
|
||
methane_stability REAL,
|
||
peak_stability REAL,
|
||
checksum TEXT
|
||
)
|
||
""")
|
||
conn.commit()
|
||
return conn, table_name
|
||
|
||
def insert_data(self, conn, data, table_name):
|
||
cursor = conn.cursor()
|
||
|
||
columns = ["timestamp","attachment_code", "device_address", "function_code", "data_length", "checksum"]
|
||
placeholders = ["?", "?", "?", "?", "?", "?"]
|
||
for i in range(len(data["valid_data_pairs"])):
|
||
# columns.append(f"data{i + 1}")
|
||
columns.append(self.sign_scale_arry[i][2])
|
||
placeholders.append("?")
|
||
|
||
columns_str = ', '.join(columns)
|
||
placeholders_str = ', '.join(placeholders)
|
||
sql = f"INSERT INTO {table_name} ({columns_str}) VALUES ({placeholders_str})"
|
||
|
||
values = [
|
||
data["timestamp"],
|
||
data["attachment_code"],
|
||
data["device_address"],
|
||
data["function_code"],
|
||
data["data_length"],
|
||
data["checksum"]
|
||
]
|
||
values.extend(data["valid_data_pairs"])
|
||
|
||
cursor.execute(sql, tuple(values))
|
||
conn.commit()
|
||
|
||
def add_to_buffer(self, parsed_data):
|
||
self.buffer.append(parsed_data)
|
||
if len(self.buffer) >= self.buffer_size:
|
||
self.flush_buffer()
|
||
elif (datetime.now() - self.last_insert_time).total_seconds() >= self.buffer_interval:
|
||
self.flush_buffer()
|
||
|
||
def flush_buffer(self):
|
||
if not self.buffer:
|
||
return
|
||
self.executor.submit(self._flush_buffer_thread)
|
||
|
||
def _flush_buffer_thread(self):
|
||
|
||
try:
|
||
grouped_data = {}
|
||
for data in self.buffer:
|
||
attachment_code = data["attachment_code"]
|
||
if attachment_code not in grouped_data:
|
||
grouped_data[attachment_code] = []
|
||
grouped_data[attachment_code].append(data)
|
||
|
||
current_month = datetime.now().strftime("%Y_%m")
|
||
db_file = f"modbus_data_{current_month}.db"
|
||
|
||
for attachment_code, data_list in grouped_data.items():
|
||
conn, table_name = self.init_database(attachment_code, db_file)
|
||
with self.db_lock: # 使用锁来同步数据库操作
|
||
for data in data_list:
|
||
self.insert_data(conn, data, table_name)
|
||
logging.debug(f"保存数据库，数据已插入{table_name}")
|
||
conn.close()
|
||
|
||
self.buffer.clear()
|
||
self.last_insert_time = datetime.now()
|
||
logging.debug("数据已保存到数据库")
|
||
except Exception as e:
|
||
logging.error(f"插入数据失败: {e}")
|
||
|
||
def auto_save_hex_to_db(self, data_bytes):
|
||
parsed_data = self.parse_modbus_data(data_bytes)
|
||
if parsed_data:
|
||
self.add_to_buffer(parsed_data)
|
||
|
||
def cleanup_old_databases(self, months_to_keep=3):
|
||
try:
|
||
current_month = datetime.now().strftime("%Y_%m")
|
||
cutoff_date = datetime.now() - timedelta(days=months_to_keep * 30)
|
||
cutoff_month = cutoff_date.strftime("%Y_%m")
|
||
|
||
for month in self.get_existing_months():
|
||
if month < cutoff_month:
|
||
db_file = f"modbus_data_{month}.db"
|
||
logging.info(f"删除旧数据库文件: {db_file}")
|
||
os.remove(db_file)
|
||
except Exception as e:
|
||
logging.error(f"清理旧数据库文件失败: {e}")
|
||
|
||
def get_existing_months(self):
|
||
import os
|
||
existing_months = []
|
||
for filename in os.listdir('.'):
|
||
if filename.startswith("modbus_data_") and filename.endswith(".db"):
|
||
month = filename[12:-3] # 提取月份部分
|
||
existing_months.append(month)
|
||
return existing_months
|
||
|
||
|
||
# 示例使用
|
||
if __name__ == "__main__":
|
||
processor = DataProcessor()
|
||
# 模拟数据解析和保存
|
||
hex_data = b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F'
|
||
processor.auto_save_hex_to_db(hex_data)
|
||
|
||
# 定期清理旧数据库文件
|
||
processor.cleanup_old_databases(months_to_keep=3)
|