lnk/microser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

完成redisstream前置生产和消费功能

Browse Source
This commit is contained in:
lnk
2026-06-30 14:19:27 +08:00
parent ae911b46e5
commit d69c823575
5 changed files with 2293 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

380
redisstream/RedisStreamMQ.cpp
View File

@@ -27,6 +27,7 @@
#include <cstdio>
#include "../log4cplus/log4.h"
#include "../mms/db_interface.h"
/*
* lnk20260622 修改：运行时切换版 Redis Streams 后端实现。
@@ -47,6 +48,10 @@ extern std::string G_MQCONSUMER_IPPORT; // Redis 模式下消费者连接地
extern std::string G_ROCKETMQ_CONSUMER; // Redis consumer group
extern std::string FRONT_INST;
// lnk20260626 修改：Redis 模式复用 RocketMq/ConsumerSecretKey 作为 Redis 密码
extern std::string G_MQCONSUMER_SECRETKEY;
extern int g_front_seg_index;
extern char subdir[128];
@@ -56,6 +61,14 @@ static const int REDIS_READ_COUNT = 16; //每次 XREADGROUP 的最大读取
static const size_t GZIP_THRESHOLD_BYTES = 1024; //超过这个大小的消息才压缩
static const char* SPOOL_DIR = "/FeProject/data/redisstream_spool"; //本地缓存目录
// lnk20260626 修改：XADD 退避与 Redis 内存保护
static const int64_t REDIS_XADD_OOM_BACKOFF_MS = 30000; // OOM 后 30 秒内不再 XADD
static const int64_t REDIS_XADD_FAIL_BACKOFF_MS = 5000; // 普通连接失败后 5 秒退避
static const int64_t REDIS_MEMORY_CHECK_INTERVAL_MS = 5000; // 每 5 秒检查一次 Redis 内存
static const int REDIS_MEMORY_HIGH_WATERMARK = 85; // Redis 内存超过 85% 后延迟 XADD
static const int64_t REDIS_SPOOL_FLUSH_INTERVAL_MS = 2000; // spool 最多 2 秒补发一次
static const int REDIS_SPOOL_FLUSH_LIMIT = 50; // 每次最多补发 50 条，防止恢复后瞬间冲击 Redis
static int64_t now_ms() //当前时间，单位毫秒
{
struct timeval tv;
@@ -151,9 +164,20 @@ static std::string gzip_compress(const std::string& input)//压缩字符串，
z_stream zs;
memset(&zs, 0, sizeof(zs));
static bool zlib_version_logged = false;
if (!zlib_version_logged) {
zlib_version_logged = true;
std::cerr << "[REDIS_STREAM][ZLIB] compile=" << ZLIB_VERSION
<< " runtime=" << zlibVersion()
<< " z_stream_size=" << sizeof(z_stream)
<< std::endl;
}
int ret = deflateInit2(&zs, Z_BEST_SPEED, Z_DEFLATED, 15 + 16, 8, Z_DEFAULT_STRATEGY);
if (ret != Z_OK) {
throw std::runtime_error("deflateInit2 failed");
char buf[64];
snprintf(buf, sizeof(buf), "deflateInit2 failed ret=%d", ret);
throw std::runtime_error(buf);
}
zs.next_in = (Bytef*)input.data();
@@ -269,9 +293,23 @@ static void encode_body(const std::string& json,
std::string& enc,
std::string& body)
{
if (json.size() >= GZIP_THRESHOLD_BYTES) {
enc = "gzip";
static bool gzip_disabled = false;
if (json.size() >= GZIP_THRESHOLD_BYTES && !gzip_disabled) {
try {
enc = "gzip";
body = base64_encode(gzip_compress(json)); //大的压缩后编码
} catch (const std::exception& e) {
std::cerr << "[REDIS_STREAM][GZIP_FALLBACK] " << e.what()
<< ", send plain body" << std::endl;
if (std::string(e.what()).find("ret=-2") != std::string::npos) {
gzip_disabled = true;
std::cerr << "[REDIS_STREAM][GZIP_DISABLED] zlib init stream error, keep sending plain body"
<< std::endl;
}
enc = "plain";
body = json;
}
} else {
enc = "plain";
body = json; //小的保持json
@@ -323,6 +361,10 @@ public:
host_ = parse_host(addr);
port_ = parse_port(addr);
db_ = parse_db(addr);
// lnk20260626 修改：Redis 模式复用 RocketMq/ConsumerSecretKey 作为密码
password_ = G_MQCONSUMER_SECRETKEY;
if (host_.empty()) host_ = "127.0.0.1";
}
@@ -389,6 +431,29 @@ private:
redisSetTimeout(ctx_, tv);
// lnk20260626 修改：Redis 密码认证
if (!password_.empty()) {
redisReply* r = (redisReply*)redisCommand(
ctx_,
"AUTH %b",
password_.data(),
password_.size());
if (!r || r->type == REDIS_REPLY_ERROR) {
std::cerr << "[REDIS_STREAM] auth failed" << std::endl;
if (r) {
if (r->str) {
std::cerr << "[REDIS_STREAM] auth error: " << r->str << std::endl;
}
freeReplyObject(r);
}
close();
return;
}
freeReplyObject(r);
}
if (db_ >= 0) {
redisReply* r = (redisReply*)redisCommand(ctx_, "SELECT %d", db_);
if (!r || (r->type == REDIS_REPLY_ERROR)) {
@@ -405,18 +470,32 @@ private:
redisContext* ctx_;
std::string addr_;
std::string host_;
std::string password_; // lnk20260626 修改：Redis AUTH 密码
int port_;
int db_;
};
// lnk20260626 修改：区分 XADD 成功、OOM、普通失败
enum RedisXAddResult {
REDIS_XADD_OK = 0,
REDIS_XADD_OOM = 1,
REDIS_XADD_FAIL = 2
};
class RedisStreamProducer { //RedisStream 生产者封装
public:
explicit RedisStreamProducer(const std::string& addr)
: xadd_pause_until_ms_(0),
last_flush_ms_(0),
last_memory_check_ms_(0),
cached_memory_high_(false),
last_pressure_log_ms_(0)
{
conn_.configure(addr);
ensure_dir(SPOOL_DIR);
if (!conn_.ping()) {
std::cerr << "[REDIS_STREAM][PRODUCER] Redis not ready, spool enabled" << std::endl;
xadd_pause_until_ms_ = now_ms() + REDIS_XADD_FAIL_BACKOFF_MS;
}
}
@@ -425,30 +504,173 @@ public:
const std::string& tag,
const std::string& key)
{
flushSpool();
const int64_t n = now_ms();
std::string enc;
std::string body; //序列化的数据json
encode_body(json, enc, body); //压缩
std::string body;
encode_body(json, enc, body);
const std::string stream = build_stream_name(topic, tag); //构造stream名称，格式：env_shortTopic
const std::string ts = to_string_i64(now_ms());
const std::string stream = build_stream_name(topic, tag);
const std::string ts = to_string_i64(n);
if (!xadd(stream, key, tag, enc, body, ts)) { //
/*
* lnk20260626 修改：
* Redis 处于退避期时，不再尝试 XADD，直接落本地 spool。
* 这就是“延迟 XADD”。
*/
if (n < xadd_pause_until_ms_) {
spool(stream, key, tag, enc, body, ts);
throw std::runtime_error("Redis XADD failed, message spooled");
logPressureOnce("[REDIS_STREAM][DELAY_XADD] redis in backoff, message spooled");
return;
}
/*
* lnk20260626 修改：
* 定期检查 Redis 内存，如果超过高水位，提前停止 XADD，避免真正 OOM。
*/
if (isRedisMemoryHigh()) {
xadd_pause_until_ms_ = now_ms() + REDIS_XADD_OOM_BACKOFF_MS;
spool(stream, key, tag, enc, body, ts);
logPressureOnce("[REDIS_STREAM][MEM_HIGH] redis memory high, delay xadd and spool message");
return;
}
/*
* lnk20260626 修改：
* spool 不要每条消息都 flush，避免恢复后瞬间冲击 Redis。
*/
if (n - last_flush_ms_ >= REDIS_SPOOL_FLUSH_INTERVAL_MS) {
flushSpool();
last_flush_ms_ = n;
}
RedisXAddResult ret = xadd(stream, key, tag, enc, body, ts);
if (ret == REDIS_XADD_OK) {
return;
}
/*
* lnk20260626 修改：
* XADD OOM 后进入退避，后续消息直接 spool。
*/
if (ret == REDIS_XADD_OOM) {
xadd_pause_until_ms_ = now_ms() + REDIS_XADD_OOM_BACKOFF_MS;
spool(stream, key, tag, enc, body, ts);
logPressureOnce("[REDIS_STREAM][OOM_BACKOFF] XADD OOM, delay xadd and spool message");
return;
}
/*
* 普通失败，例如网络断开，也进入短退避。
*/
xadd_pause_until_ms_ = now_ms() + REDIS_XADD_FAIL_BACKOFF_MS;
spool(stream, key, tag, enc, body, ts);
logPressureOnce("[REDIS_STREAM][XADD_FAIL_BACKOFF] XADD failed, message spooled");
}
void shutdown() {}
private:
bool xadd(const std::string& stream,
const std::string& key,
const std::string& tag,
const std::string& enc,
const std::string& body,
const std::string& ts)
void logPressureOnce(const char* msg)
{
const int64_t n = now_ms();
/*
* 压力日志最多 10 秒打一条，避免 OOM 时刷屏。
*/
if (n - last_pressure_log_ms_ >= 10000) {
last_pressure_log_ms_ = n;
std::cerr << msg << std::endl;
DIY_ERRORLOG_CODE("process", 0, LOG_CODE_MQ,
"【WARN】前置的%s%d号进程 Redis Stream写入延迟，消息已落本地缓存",
get_front_msg_from_subdir(), g_front_seg_index);
}
}
static bool parse_info_number(const std::string& info,
const std::string& key,
long long& value)
{
value = 0;
std::string pattern = key + ":";
size_t pos = info.find(pattern);
if (pos == std::string::npos) return false;
pos += pattern.size();
size_t end = info.find_first_of("\r\n", pos);
std::string num = info.substr(pos, end == std::string::npos ? std::string::npos : end - pos);
if (num.empty()) return false;
value = atoll(num.c_str());
return value > 0;
}
bool isRedisMemoryHigh()
{
const int64_t n = now_ms();
if (n - last_memory_check_ms_ < REDIS_MEMORY_CHECK_INTERVAL_MS) {
return cached_memory_high_;
}
last_memory_check_ms_ = n;
cached_memory_high_ = false;
redisReply* r = conn_.command("INFO memory");
if (!r) {
return false;
}
if (r->type != REDIS_REPLY_STRING || !r->str) {
freeReplyObject(r);
return false;
}
std::string info(r->str, r->len);
freeReplyObject(r);
long long used_memory = 0;
long long maxmemory = 0;
if (!parse_info_number(info, "used_memory", used_memory)) {
return false;
}
if (!parse_info_number(info, "maxmemory", maxmemory)) {
return false;
}
/*
* maxmemory=0 表示 Redis 没有限制内存。
* 你的环境设置了 2G，所以这里应该能读到 maxmemory。
*/
if (maxmemory <= 0) {
return false;
}
if (used_memory * 100 >= maxmemory * REDIS_MEMORY_HIGH_WATERMARK) {
cached_memory_high_ = true;
std::cerr << "[REDIS_STREAM][MEM_HIGH] used_memory=" << used_memory
<< " maxmemory=" << maxmemory
<< " percent=" << (used_memory * 100 / maxmemory)
<< "% watermark=" << REDIS_MEMORY_HIGH_WATERMARK
<< "%" << std::endl;
}
return cached_memory_high_;
}
RedisXAddResult xadd(const std::string& stream,
const std::string& key,
const std::string& tag,
const std::string& enc,
const std::string& body,
const std::string& ts)
{
redisReply* r = conn_.command(
"XADD %b * key %b tag %b enc %b body %b ts %b",
@@ -459,14 +681,36 @@ private:
body.data(), body.size(),
ts.data(), ts.size());
if (!r) return false;
bool ok = (r->type == REDIS_REPLY_STRING);
if (!ok && r->type == REDIS_REPLY_ERROR) {
std::cerr << "[REDIS_STREAM][XADD_ERROR] " << (r->str ? r->str : "") << std::endl;
if (!r) {
return REDIS_XADD_FAIL;
}
if (r->type == REDIS_REPLY_STRING) {
freeReplyObject(r);
return REDIS_XADD_OK;
}
if (r->type == REDIS_REPLY_ERROR) {
std::string err = r->str ? r->str : "";
std::cerr << "[REDIS_STREAM][XADD_ERROR] " << err << std::endl;
freeReplyObject(r);
/*
* lnk20260626 修改：
* noeviction + maxmemory 满时，Redis 会返回 OOM。
*/
if (err.find("OOM") != std::string::npos ||
err.find("maxmemory") != std::string::npos ||
err.find("command not allowed") != std::string::npos) {
return REDIS_XADD_OOM;
}
return REDIS_XADD_FAIL;
}
freeReplyObject(r);
return ok;
return REDIS_XADD_FAIL;
}
void spool(const std::string& stream,
@@ -477,8 +721,15 @@ private:
const std::string& ts)
{
ensure_dir(SPOOL_DIR);
std::string fn = std::string(SPOOL_DIR) + "/" + to_string_i64(now_ms()) + "_" +
shell_safe_name(stream) + "_" + shell_safe_name(key) + ".msg.tmp";
char uniq[128];
snprintf(uniq, sizeof(uniq), "%lld_%d_%llu",
(long long)now_ms(),
(int)getpid(),
(unsigned long long)rand());
std::string fn = std::string(SPOOL_DIR) + "/" + uniq + "_" +
shell_safe_name(stream) + "_" + shell_safe_name(key) + ".msg.tmp";
std::string done = fn.substr(0, fn.size() - 4);
std::ofstream out(fn.c_str(), std::ios::out | std::ios::binary);
@@ -533,6 +784,18 @@ private:
void flushSpool()
{
const int64_t n = now_ms();
if (n < xadd_pause_until_ms_) {
return;
}
if (isRedisMemoryHigh()) {
xadd_pause_until_ms_ = now_ms() + REDIS_XADD_OOM_BACKOFF_MS;
logPressureOnce("[REDIS_STREAM][SPOOL_DELAY] redis memory high, stop flushing spool");
return;
}
DIR* dir = opendir(SPOOL_DIR);
if (!dir) return;
@@ -548,22 +811,52 @@ private:
std::sort(files.begin(), files.end());
int flushed = 0;
for (size_t i = 0; i < files.size(); ++i) {
if (flushed >= REDIS_SPOOL_FLUSH_LIMIT) {
break;
}
std::string stream, key, tag, enc, body, ts;
if (!parse_spool_file(files[i], stream, key, tag, enc, body, ts)) {
continue;
}
if (xadd(stream, key, tag, enc, body, ts)) {
RedisXAddResult ret = xadd(stream, key, tag, enc, body, ts);
if (ret == REDIS_XADD_OK) {
unlink(files[i].c_str());
++flushed;
std::cout << "[REDIS_STREAM][SPOOL_FLUSHED] " << files[i] << std::endl;
} else {
continue;
}
if (ret == REDIS_XADD_OOM) {
xadd_pause_until_ms_ = now_ms() + REDIS_XADD_OOM_BACKOFF_MS;
logPressureOnce("[REDIS_STREAM][SPOOL_OOM] XADD OOM while flushing spool");
break;
}
xadd_pause_until_ms_ = now_ms() + REDIS_XADD_FAIL_BACKOFF_MS;
logPressureOnce("[REDIS_STREAM][SPOOL_FAIL] XADD failed while flushing spool");
break;
}
}
private:
RedisConnection conn_;
// lnk20260626 修改：延迟 XADD 控制
int64_t xadd_pause_until_ms_;
int64_t last_flush_ms_;
// lnk20260626 修改：Redis 内存高水位缓存
int64_t last_memory_check_ms_;
bool cached_memory_high_;
// lnk20260626 修改：压力日志限频
int64_t last_pressure_log_ms_;
};
class RedisStreamConsumer { //RedisStream 消费者封装
@@ -574,12 +867,16 @@ public:
conn_.configure(addr);
if (group_.empty()) group_ = "front_default";
char buf[64];
snprintf(buf, sizeof(buf), "_%s_%d", subdir, g_front_seg_index);
group_ += buf;
int64_t pid = (int64_t)getpid();
int64_t start_ms = now_ms();
consumer_ = group_ + "_" + to_string_i64((int64_t)getpid()) + "_" + to_string_i64(now_ms());
char buf[128];
snprintf(buf, sizeof(buf), "_%s_%d_%lld_%lld",
subdir, g_front_seg_index,
(long long)pid, (long long)start_ms);
group_ += buf;
consumer_ = group_;
}
void subscribe(const std::string& topic,
@@ -680,6 +977,7 @@ private:
std::string enc = getField(fields, "enc");
std::string bodyRaw = getField(fields, "body");
std::string body;
if (!decode_body(enc, bodyRaw, body)) {
throw std::runtime_error("decode body failed");
}
@@ -688,11 +986,27 @@ private:
std::string tag = getField(fields, "tag");
std::string ts = getField(fields, "ts");
if (tag.empty()) tag = sub.tag;
if (tag.empty()) {
tag = sub.tag;
}
int64_t born = ts.empty() ? now_ms() : atoll(ts.c_str());
return rocketmq::MQMessageExt(sub.topic, tag, key, body, born, id);
rocketmq::MQMessageExt msg;
msg.setTopic(sub.topic);
msg.setTags(tag);
msg.setKeys(key);
msg.setBody(body);
msg.setBornTimestamp(born);
/*
* 如果你的 RocketMQ MQMessageExt.h 里有 setMsgId，就打开。
* 如果没有这个函数，就注释掉。
*/
// msg.setMsgId(id);
return msg;
}
MessageCallBack findCallback(const std::string& topic,