Fatal glibc error: CPU does not support x86-64-v2

# 缓存 API 响应
redis-cli SET "user:${id}" '{"id":123,"name":"John"}' EX 3600

# 实时通知
redis-cli PUBLISH notifications '{"userId":123,"message":"Hello"}'

# 使用 Bull/BullMQ（这里展示 Redis CLI 的基础操作）
redis-cli LPUSH queue:send-email '{"to":"user@example.com","subject":"Hi"}'

Postgres 备份 ✅
Redis 备份 ❓（RDB？AOF？两者都要？）
Postgres 监控 ✅
Redis 监控 ❓
Postgres 故障转移 ✅
Redis Sentinel/Cluster ❓

Postgres 备份 ✅
Postgres 监控 ✅
Postgres 故障转移 ✅

# 更新数据库
psql -c "UPDATE users SET name = 'John' WHERE id = 123;"

# 使缓存失效
redis-cli DEL "user:123"

# ⚠️ 如果 Redis 宕机了怎么办？
# ⚠️ 如果这个操作失败了怎么办？
# 现在缓存和数据库不同步了

redis-cli SET "session:abc123" '{"userId":123,"role":"admin"}' EX 3600

CREATE UNLOGGED TABLE cache (
  key TEXT PRIMARY KEY,
  value JSONB NOT NULL,
  expires_at TIMESTAMPTZ NOT NULL
);

CREATE INDEX idx_cache_expires ON cache(expires_at);

INSERT INTO cache (key, value, expires_at)
VALUES ('user:123', '{"id":123,"name":"John"}'::jsonb, NOW() + INTERVAL '1 hour')
ON CONFLICT (key) DO UPDATE
  SET value = EXCLUDED.value,
      expires_at = EXCLUDED.expires_at;

SELECT value FROM cache
WHERE key = 'user:123' AND expires_at > NOW();

DELETE FROM cache WHERE expires_at < NOW();

Redis SET: 0.05ms
Postgres UNLOGGED INSERT: 0.08ms

# 发布者
redis-cli PUBLISH notifications '{"userId":123,"msg":"Hello"}'

# 订阅者（在另一个终端）
redis-cli SUBSCRIBE notifications

-- 发布者
NOTIFY notifications, '{"userId": 123, "msg": "Hello"}';

// 订阅者（Java with PostgreSQL JDBC）
import org.postgresql.PGConnection;
import org.postgresql.PGNotification;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.Statement;
import java.util.Properties;
import com.fasterxml.jackson.databind.ObjectMapper;

// 建立连接
String url = System.getenv("DATABASE_URL");
Properties props = new Properties();
Connection conn = DriverManager.getConnection(url, props);
Statement stmt = conn.createStatement();
stmt.execute("LISTEN notifications");
stmt.close();

// 获取 PGConnection 以接收通知
PGConnection pgConn = conn.unwrap(PGConnection.class);
org.postgresql.PGNotification[] notifications = pgConn.getNotifications();

// 在单独的线程中监听通知
new Thread(() -> {
    while (true) {
        try {
            PGNotification[] notifications = pgConn.getNotifications();
            if (notifications != null) {
                for (PGNotification notification : notifications) {
                    String payload = notification.getParameter();
                    ObjectMapper mapper = new ObjectMapper();
                    Map<String, Object> data = mapper.readValue(payload, Map.class);
                    System.out.println(data);
                }
            }
            Thread.sleep(500);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}).start();

Redis pub/sub 延迟：1-2ms
Postgres NOTIFY 延迟：2-5ms

# 当新日志到达时
psql -c "INSERT INTO logs ..."
redis-cli PUBLISH logs:new '{"id":123,"message":"..."}'

# 前端监听
redis-cli SUBSCRIBE logs:new

CREATE FUNCTION notify_new_log() RETURNS TRIGGER AS $$
BEGIN
  PERFORM pg_notify('logs_new', row_to_json(NEW)::text);
  RETURN NEW;
END;
$$ LANGUAGE plpgsql;

CREATE TRIGGER log_inserted
AFTER INSERT ON logs
FOR EACH ROW EXECUTE FUNCTION notify_new_log();

// 前端（通过 SSE）- Spring Boot 示例
import org.springframework.http.MediaType;
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
import org.postgresql.PGConnection;
import org.postgresql.PGNotification;
import java.sql.Connection;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

@GetMapping(value = "/logs/stream", produces = MediaType.TEXT_EVENT_STREAM_VALUE)
public SseEmitter streamLogs() {
    SseEmitter emitter = new SseEmitter(Long.MAX_VALUE);
    ExecutorService executor = Executors.newSingleThreadExecutor();
    
    executor.execute(() -> {
        try {
            Connection conn = dataSource.getConnection();
            Statement stmt = conn.createStatement();
            stmt.execute("LISTEN logs_new");
            stmt.close();
            
            PGConnection pgConn = conn.unwrap(PGConnection.class);
            
            while (true) {
                PGNotification[] notifications = pgConn.getNotifications();
                if (notifications != null) {
                    for (PGNotification notification : notifications) {
                        String payload = notification.getParameter();
                        emitter.send(SseEmitter.event()
                            .data("data: " + payload + "\n\n"));
                    }
                }
                Thread.sleep(100);
            }
        } catch (Exception e) {
            emitter.completeWithError(e);
        }
    });
    
    return emitter;
}

# 入队
redis-cli LPUSH queue:send-email '{"to":"user@example.com","subject":"Hi"}'

# 出队（使用阻塞操作）
redis-cli BRPOP queue:send-email 5

CREATE TABLE jobs (
  id BIGSERIAL PRIMARY KEY,
  queue TEXT NOT NULL,
  payload JSONB NOT NULL,
  attempts INT DEFAULT 0,
  max_attempts INT DEFAULT 3,
  scheduled_at TIMESTAMPTZ DEFAULT NOW(),
  created_at TIMESTAMPTZ DEFAULT NOW()
);

CREATE INDEX idx_jobs_queue ON jobs(queue, scheduled_at) 
WHERE attempts < max_attempts;

INSERT INTO jobs (queue, payload)
VALUES ('send-email', '{"to": "user@example.com", "subject": "Hi"}'::jsonb);

WITH next_job AS (
  SELECT id FROM jobs
  WHERE queue = 'send-email'
    AND attempts < max_attempts
    AND scheduled_at <= NOW()
  ORDER BY scheduled_at
  LIMIT 1
  FOR UPDATE SKIP LOCKED
)
UPDATE jobs
SET attempts = attempts + 1
FROM next_job
WHERE jobs.id = next_job.id
RETURNING *;

Redis BRPOP: 0.1ms
Postgres SKIP LOCKED: 0.3ms

BEGIN;

-- 取出 1 个 pending 状态的任务，加锁并跳过已锁定的
UPDATE tasks 
SET status = 'processing'
WHERE id = (
    SELECT id 
    FROM tasks 
    WHERE status = 'pending'
    ORDER BY id
    LIMIT 1  --- 可以LIMIT 10来达到批量拉取的效果
    FOR UPDATE SKIP LOCKED  -- 👈 核心在这里
)
RETURNING *;

COMMIT;

--- 需要考虑对僵尸任务的释放
-- 任务处理超时 5 分钟，自动释放
UPDATE tasks 
SET status = 'pending'
WHERE status = 'processing'
AND updated_at < NOW() - INTERVAL '5 minutes';

# 检查并增加计数
redis-cli INCR "ratelimit:${userId}"
redis-cli EXPIRE "ratelimit:${userId}" 60

# 检查是否超过限制
redis-cli GET "ratelimit:${userId}"

CREATE TABLE rate_limits (
  user_id INT PRIMARY KEY,
  request_count INT DEFAULT 0,
  window_start TIMESTAMPTZ DEFAULT NOW()
);

-- 检查并增加
WITH current AS (
  SELECT 
    request_count,
    CASE 
      WHEN window_start < NOW() - INTERVAL '1 minute'
      THEN 1  -- 重置计数器
      ELSE request_count + 1
    END AS new_count
  FROM rate_limits
  WHERE user_id = 123
  FOR UPDATE
)
UPDATE rate_limits
SET 
  request_count = (SELECT new_count FROM current),
  window_start = CASE
    WHEN window_start < NOW() - INTERVAL '1 minute'
    THEN NOW()
    ELSE window_start
  END
WHERE user_id = 123
RETURNING request_count;

CREATE TABLE api_requests (
  user_id INT NOT NULL,
  created_at TIMESTAMPTZ DEFAULT NOW()
);

-- 检查速率限制
SELECT COUNT(*) FROM api_requests
WHERE user_id = 123
  AND created_at > NOW() - INTERVAL '1 minute';

-- 如果在限制内，插入
INSERT INTO api_requests (user_id) VALUES (123);

-- 定期清理旧请求
DELETE FROM api_requests WHERE created_at < NOW() - INTERVAL '5 minutes';

redis-cli SET "session:${sessionId}" '{"userId":123,"role":"admin"}' EX 86400

CREATE TABLE sessions (
  id TEXT PRIMARY KEY,
  data JSONB NOT NULL,
  expires_at TIMESTAMPTZ NOT NULL
);

CREATE INDEX idx_sessions_expires ON sessions(expires_at);

-- 插入/更新
INSERT INTO sessions (id, data, expires_at)
VALUES ('abc123', '{"userId":123,"role":"admin"}'::jsonb, NOW() + INTERVAL '24 hours')
ON CONFLICT (id) DO UPDATE
  SET data = EXCLUDED.data,
      expires_at = EXCLUDED.expires_at;

-- 读取
SELECT data FROM sessions
WHERE id = 'abc123' AND expires_at > NOW();

-- 查找特定用户的所有会话
SELECT * FROM sessions
WHERE data->>'userId' = '123';

-- 查找具有特定角色的会话
SELECT * FROM sessions
WHERE data->'user'->>'role' = 'admin';

psql -c "INSERT INTO posts ..."                    # 2ms
redis-cli DEL "posts:latest"                        # 1ms（网络跳转）
redis-cli PUBLISH posts:new '{"id":123}'            # 1ms（网络跳转）
# 总计：~4ms

BEGIN;
INSERT INTO posts ...;                              -- 2ms
DELETE FROM cache WHERE key = 'posts:latest';      -- 0.1ms（同一连接）
NOTIFY posts_new, '...';                            -- 0.1ms（同一连接）
COMMIT;
-- 总计：~2.2ms

Redis: 100,000+ 操作/秒（单实例）
Postgres: 10,000-50,000 操作/秒

-- Postgres 中的排行榜（更慢）
SELECT user_id, score
FROM leaderboard
ORDER BY score DESC
LIMIT 10;

-- vs Redis
redis-cli ZREVRANGE leaderboard 0 9 WITHSCORES

// 写入两者
jedis.set(key, value);
jdbcTemplate.update("INSERT INTO cache ...", key, value);

// 从 Redis 读取（仍然是主要的）
String data = jedis.get(key);

// 先尝试 Postgres
String data = jdbcTemplate.queryForObject(
    "SELECT value FROM cache WHERE key = ? AND expires_at > NOW()",
    String.class, key);

// 回退到 Redis
if (data == null || data.isEmpty()) {
    data = jedis.get(key);
}

// 只写入 Postgres
jdbcTemplate.update("INSERT INTO cache (key, value, expires_at) VALUES (?, ?, ?)",
    key, value, LocalDateTime.now().plusHours(1));

# 关闭 Redis
# 观察错误
# 没有破坏？成功！

// PostgresCache.java
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
import java.time.LocalDateTime;
import java.util.Map;

@Component
public class PostgresCache {
    private final JdbcTemplate jdbcTemplate;
    
    public PostgresCache(JdbcTemplate jdbcTemplate) {
        this.jdbcTemplate = jdbcTemplate;
    }
    
    public String get(String key) {
        return jdbcTemplate.queryForObject(
            "SELECT value FROM cache WHERE key = ? AND expires_at > NOW()",
            String.class, key);
    }
    
    public void set(String key, String value, int ttlSeconds) {
        jdbcTemplate.update(
            "INSERT INTO cache (key, value, expires_at) " +
            "VALUES (?, ?::jsonb, NOW() + INTERVAL ? || ' seconds') " +
            "ON CONFLICT (key) DO UPDATE " +
            "SET value = EXCLUDED.value, expires_at = EXCLUDED.expires_at",
            key, value, ttlSeconds);
    }
    
    public void set(String key, String value) {
        set(key, value, 3600);
    }
    
    public void delete(String key) {
        jdbcTemplate.update("DELETE FROM cache WHERE key = ?", key);
    }
    
    public void cleanup() {
        jdbcTemplate.update("DELETE FROM cache WHERE expires_at < NOW()");
    }
}

// PostgresPubSub.java
import org.postgresql.PGConnection;
import org.postgresql.PGNotification;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.sql.Connection;
import java.sql.Statement;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.function.Consumer;

@Component
public class PostgresPubSub {
    private final JdbcTemplate jdbcTemplate;
    private final ObjectMapper objectMapper;
    private final Map<String, Connection> listeners = new ConcurrentHashMap<>();
    private final ExecutorService executor = Executors.newCachedThreadPool();
    
    public PostgresPubSub(JdbcTemplate jdbcTemplate) {
        this.jdbcTemplate = jdbcTemplate;
        this.objectMapper = new ObjectMapper();
    }
    
    public void publish(String channel, Object message) throws Exception {
        String payload = objectMapper.writeValueAsString(message);
        jdbcTemplate.update("SELECT pg_notify(?, ?)", channel, payload);
    }
    
    public void subscribe(String channel, Consumer<Map<String, Object>> callback) {
        executor.execute(() -> {
            try {
                Connection conn = jdbcTemplate.getDataSource().getConnection();
                Statement stmt = conn.createStatement();
                stmt.execute("LISTEN " + channel);
                stmt.close();
                
                listeners.put(channel, conn);
                
                PGConnection pgConn = conn.unwrap(PGConnection.class);
                while (listeners.containsKey(channel)) {
                    PGNotification[] notifications = pgConn.getNotifications();
                    if (notifications != null) {
                        for (PGNotification notification : notifications) {
                            if (notification.getName().equals(channel)) {
                                Map<String, Object> data = objectMapper.readValue(
                                    notification.getParameter(), Map.class);
                                callback.accept(data);
                            }
                        }
                    }
                    Thread.sleep(100);
                }
            } catch (Exception e) {
                e.printStackTrace();
            }
        });
    }
    
    public void unsubscribe(String channel) {
        Connection conn = listeners.remove(channel);
        if (conn != null) {
            try {
                Statement stmt = conn.createStatement();
                stmt.execute("UNLISTEN " + channel);
                stmt.close();
                conn.close();
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }
}

// PostgresQueue.java
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.stereotype.Component;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.time.LocalDateTime;
import java.util.Map;

@Component
public class PostgresQueue {
    private final JdbcTemplate jdbcTemplate;
    private final ObjectMapper objectMapper;
    
    public PostgresQueue(JdbcTemplate jdbcTemplate) {
        this.jdbcTemplate = jdbcTemplate;
        this.objectMapper = new ObjectMapper();
    }
    
    public void enqueue(String queue, Map<String, Object> payload, LocalDateTime scheduledAt) {
        try {
            String payloadJson = objectMapper.writeValueAsString(payload);
            jdbcTemplate.update(
                "INSERT INTO jobs (queue, payload, scheduled_at) VALUES (?, ?::jsonb, ?)",
                queue, payloadJson, scheduledAt);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }
    
    public void enqueue(String queue, Map<String, Object> payload) {
        enqueue(queue, payload, LocalDateTime.now());
    }
    
    public Job dequeue(String queue) {
        String sql = "WITH next_job AS (" +
            "  SELECT id FROM jobs " +
            "  WHERE queue = ? " +
            "    AND attempts < max_attempts " +
            "    AND scheduled_at <= NOW() " +
            "  ORDER BY scheduled_at " +
            "  LIMIT 1 " +
            "  FOR UPDATE SKIP LOCKED " +
            ") " +
            "UPDATE jobs " +
            "SET attempts = attempts + 1 " +
            "FROM next_job " +
            "WHERE jobs.id = next_job.id " +
            "RETURNING jobs.*";
        
        return jdbcTemplate.queryForObject(sql, new JobRowMapper(), queue);
    }
    
    public void complete(Long jobId) {
        jdbcTemplate.update("DELETE FROM jobs WHERE id = ?", jobId);
    }
    
    public void fail(Long jobId, Exception error) {
        try {
            String errorJson = objectMapper.writeValueAsString(Map.of("error", error.getMessage()));
            jdbcTemplate.update(
                "UPDATE jobs " +
                "SET attempts = max_attempts, " +
                "    payload = payload || ?::jsonb " +
                "WHERE id = ?",
                errorJson, jobId);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }
    
    private static class JobRowMapper implements RowMapper<Job> {
        @Override
        public Job mapRow(ResultSet rs, int rowNum) throws SQLException {
            Job job = new Job();
            job.setId(rs.getLong("id"));
            job.setQueue(rs.getString("queue"));
            job.setPayload(rs.getString("payload"));
            job.setAttempts(rs.getInt("attempts"));
            job.setMaxAttempts(rs.getInt("max_attempts"));
            job.setScheduledAt(rs.getTimestamp("scheduled_at").toLocalDateTime());
            job.setCreatedAt(rs.getTimestamp("created_at").toLocalDateTime());
            return job;
        }
    }
    
    public static class Job {
        private Long id;
        private String queue;
        private String payload;
        private Integer attempts;
        private Integer maxAttempts;
        private LocalDateTime scheduledAt;
        private LocalDateTime createdAt;
        
        // Getters and Setters
        public Long getId() { return id; }
        public void setId(Long id) { this.id = id; }
        public String getQueue() { return queue; }
        public void setQueue(String queue) { this.queue = queue; }
        public String getPayload() { return payload; }
        public void setPayload(String payload) { this.payload = payload; }
        public Integer getAttempts() { return attempts; }
        public void setAttempts(Integer attempts) { this.attempts = attempts; }
        public Integer getMaxAttempts() { return maxAttempts; }
        public void setMaxAttempts(Integer maxAttempts) { this.maxAttempts = maxAttempts; }
        public LocalDateTime getScheduledAt() { return scheduledAt; }
        public void setScheduledAt(LocalDateTime scheduledAt) { this.scheduledAt = scheduledAt; }
        public LocalDateTime getCreatedAt() { return createdAt; }
        public void setCreatedAt(LocalDateTime createdAt) { this.createdAt = createdAt; }
    }
}

// Spring Boot 配置（application.yml）
spring:
  datasource:
    url: jdbc:postgresql://localhost:5432/mydb
    username: user
    password: password
    hikari:
      maximum-pool-size: 20  # 最大连接数
      minimum-idle: 5        # 最小空闲连接数
      connection-timeout: 2000  # 连接超时（毫秒）
      idle-timeout: 30000    # 空闲超时（毫秒）
      max-lifetime: 1800000  # 连接最大生命周期（毫秒）

// 或者使用 Java 配置
@Configuration
public class DataSourceConfig {
    @Bean
    public DataSource dataSource() {
        HikariConfig config = new HikariConfig();
        config.setJdbcUrl("jdbc:postgresql://localhost:5432/mydb");
        config.setUsername("user");
        config.setPassword("password");
        config.setMaximumPoolSize(20);
        config.setMinimumIdle(5);
        config.setConnectionTimeout(2000);
        config.setIdleTimeout(30000);
        return new HikariDataSource(config);
    }
}

CREATE INDEX CONCURRENTLY idx_cache_key ON cache(key) WHERE expires_at > NOW();
CREATE INDEX CONCURRENTLY idx_jobs_pending ON jobs(queue, scheduled_at) 
  WHERE attempts < max_attempts;

# postgresql.conf
shared_buffers = 2GB           # RAM 的 25%
effective_cache_size = 6GB     # RAM 的 75%
work_mem = 50MB                # 用于复杂查询
maintenance_work_mem = 512MB   # 用于 VACUUM

-- 每天运行
VACUUM ANALYZE cache;
VACUUM ANALYZE jobs;

-- 或启用 autovacuum（推荐）
ALTER TABLE cache SET (autovacuum_vacuum_scale_factor = 0.1);

DELIMITER $$

CREATE PROCEDURE SimpleSelectOne()
BEGIN
    SELECT 1 AS result;
    END$$

DELIMITER ;

select * from performance_schema.events_statements_history where EVENT_NAME like 'statement/sp%' \G;

SELECT 
    EVENT_ID AS call_event_id,
    OBJECT_SCHEMA AS proc_schema,
    OBJECT_NAME AS proc_name,
    SQL_TEXT
FROM performance_schema.events_statements_history
WHERE SQL_TEXT LIKE 'CALL%';

select count(distinct left(city, 3)) /  count(*) as sel3,
count(distinct left(city, 4)) /  count(*) as sel4,
count(distinct left(city, 5)) /  count(*) as sel5
from city

alter table city add key (city(7))

检查 name 条件是否使用了函数或类型转换，如 WHERE LOWER(name) = 'xxx' 或 WHERE name = 123（当 name 是字符串类型时）
检查是否使用了 !=、NOT IN 等无法使用索引的操作符

如果 name 的值非常集中（如90%的行都有相同的 name 值），优化器可能认为全表扫描比索引扫描更高效

MySQL 的优化器依赖统计信息做决策，如果统计信息过时，可能导致错误选择执行计划

如果满足 name = xxx 条件的行数超过表的约30%，优化器可能选择全表扫描

-- 最小权限原则
CREATE USER 'backup'@'%' IDENTIFIED BY 'StrongPass!2026';
GRANT SELECT, RELOAD, SHOW DATABASES, LOCK TABLES, PROCESS ON *.* TO 'backup'@'%';
FLUSH PRIVILEGES;

#!/bin/bash
# 安全 mydumper 备份脚本 - 生产环境

set -euo pipefail

BACKUP_DIR="/backup/mysql/mydumper"
DATE=$(date +%Y%m%d_%H%M)
LOG_FILE="/var/log/mydumper_backup.log"
MYSQL_HOST="127.0.0.1"
MYSQL_USER="backup"
MYSQL_PASS="StrongPass!2026"
ENCRYPTION_KEY="/etc/mysql/backup.key"  # AES-256 密钥文件

# 创建目录
mkdir -p ${BACKUP_DIR}/${DATE}

# 记录开始时间
echo "[$(date)] Starting mydumper backup..." >> $LOG_FILE

# 执行备份（压缩 + 加密 + 并行）
mydumper \
  --host=${MYSQL_HOST} \
  --user=${MYSQL_USER} \
  --password=${MYSQL_PASS} \
  --outputdir=${BACKUP_DIR}/${DATE} \
  --compress=gzip \          # 压缩节省空间
  --encrypt=AES256 \         # 加密备份文件
  --encrypt-key-file=${ENCRYPTION_KEY} \
  --threads=8 \              # 根据 CPU 调整
  --trx-consistency-only \   # 仅保证事务一致性（不锁表）
  --verbose=3 \
  >> $LOG_FILE 2>&1

# 验证备份完整性（检查 metadata 文件）
if [ ! -f "${BACKUP_DIR}/${DATE}/metadata" ]; then
  echo "[$(date)] ERROR: Backup failed - metadata missing!" >> $LOG_FILE
  exit 1
fi

# 清理 7 天前备份
find ${BACKUP_DIR} -maxdepth 1 -type d -mtime +7 -exec rm -rf {} \;

echo "[$(date)] Backup completed successfully." >> $LOG_FILE

# 解密并恢复
myloader \
  --host=127.0.0.1 \
  --user=restore_user \
  --password='RestorePass!2026' \
  --directory=/backup/mysql/mydumper/20260122_1400 \
  --decrypt=AES256 \
  --decrypt-key-file=/etc/mysql/backup.key \
  --threads=8 \
  --overwrite-tables \
  --verbose=3

CREATE USER 'xtrabackup'@'localhost' IDENTIFIED BY 'XbkPass!2026';
GRANT RELOAD, PROCESS, LOCK TABLES, REPLICATION CLIENT, SHOW DATABASES ON *.* TO 'xtrabackup'@'localhost';
FLUSH PRIVILEGES;

#!/bin/bash
# xtrabackup 全量备份 - 生产安全版

set -euo pipefail

BACKUP_BASE="/backup/mysql/xtrabackup"
DATE=$(date +%Y%m%d)
FULL_BACKUP_DIR="${BACKUP_BASE}/full_${DATE}"
LOG_FILE="/var/log/xtrabackup_full.log"
ENCRYPTION_KEY="/etc/mysql/xbk.key"

mkdir -p $FULL_BACKUP_DIR

# 流式备份到 xbstream + 压缩 + 加密
xtrabackup \
  --user=xtrabackup \
  --password=XbkPass!2026 \
  --backup \
  --target-dir=$FULL_BACKUP_DIR \
  --stream=xbstream \
  --compress=zstd \          # zstd 比 gzip 更快
  --compress-threads=4 \
  --encrypt=AES256 \
  --encrypt-key-file=$ENCRYPTION_KEY \
  --encrypt-threads=4 \
  | ssh backup-server "cat > ${FULL_BACKUP_DIR}/full.xbstream.zst.enc"

# 记录 binlog 位置（用于 PITR）
ssh backup-server "xtrabackup --decrypt=AES256 --encrypt-key-file=$ENCRYPTION_KEY --target-dir=$FULL_BACKUP_DIR && xtrabackup --decompress --target-dir=$FULL_BACKUP_DIR && xtrabackup --prepare --target-dir=$FULL_BACKUP_DIR"

echo "Full backup completed: $DATE" >> $LOG_FILE

# 基于上周日全量做增量
xtrabackup \
  --user=xtrabackup \
  --password=XbkPass!2026 \
  --backup \
  --target-dir=/tmp/inc_$(date +%Y%m%d) \
  --incremental-basedir=/backup/mysql/xtrabackup/full_20260119 \
  --stream=xbstream \
  --compress=zstd \
  --encrypt=AES256 \
  --encrypt-key-file=/etc/mysql/xbk.key \
  | ssh backup-server "cat > /backup/mysql/xtrabackup/inc_$(date +%Y%m%d).xbstream.zst.enc"

scp backup-server:/backup/mysql/xtrabackup/full_20260119.xbstream.zst.enc /restore/
xtrabackup --decrypt=AES256 --encrypt-key-file=/etc/mysql/xbk.key --target-dir=/restore/full
xtrabackup --decompress --target-dir=/restore/full

xtrabackup --decrypt=... --decompress=... --target-dir=/restore/inc_20260120
xtrabackup --prepare --apply-log-only --target-dir=/restore/full
xtrabackup --prepare --target-dir=/restore/full --incremental-dir=/restore/inc_20260120

xtrabackup --prepare --target-dir=/restore/full
rsync -avrP /restore/full/ /var/lib/mysql/
chown -R mysql:mysql /var/lib/mysql
systemctl start mysqld

-- 查看备份的 binlog 位置
cat /restore/full/xtrabackup_binlog_info

-- 用 mysqlbinlog 恢复到指定时间点
mysqlbinlog --start-position=12345 --stop-datetime="2026-01-22 14:00:00" binlog.000001 | mysql -u root -p

sudo dmesg -T | grep -i "killed process"

-- query time
show status

-- MySQL 运行状态与内存概览
SELECT 
  -- Uptime
  CONCAT(
    FLOOR(VARIABLE_VALUE / 86400), 'd ',
    FLOOR((VARIABLE_VALUE % 86400) / 3600), 'h ',
    FLOOR((VARIABLE_VALUE % 3600) / 60), 'm'
  ) AS uptime,
  
  -- 内存配置
  FORMAT(@@innodb_buffer_pool_size / 1024 / 1024, 2) AS ibp_mb,
  FORMAT(@@key_buffer_size / 1024 / 1024, 2) AS key_buffer_mb,
  
  -- 连接数
  @@max_connections AS max_conn,
  VARIABLE_VALUE AS current_conn,
  
  -- 估算峰值内存 (MB)
  FORMAT((
    @@innodb_buffer_pool_size + 
    @@key_buffer_size + 
    (@@sort_buffer_size + @@read_buffer_size + @@join_buffer_size) * @@max_connections
  ) / 1024 / 1024, 2) AS est_peak_memory_mb

FROM information_schema.GLOBAL_STATUS 
WHERE VARIABLE_NAME IN ('Uptime', 'Threads_connected')
LIMIT 1;

INSERT INTO t_dp_i_import_collection_plan(xxx)   --- 隐藏了
SELECT xxx   --- 隐藏了
FROM import_collection_plan as t
where report_date = '2025-01-31'

SELECT
    TABLE_NAME AS `Table`,
    ENGINE AS `Engine`,
    ROUND((DATA_LENGTH + INDEX_LENGTH) / 1024 / 1024, 2) AS `Size_MB`,
    ROUND(DATA_LENGTH / 1024 / 1024, 2) AS `Data_MB`,
    ROUND(INDEX_LENGTH / 1024 / 1024, 2) AS `Index_MB`,
    TABLE_ROWS AS `Est_Row_Count`
FROM
    information_schema.TABLES
WHERE
    TABLE_SCHEMA = 'xxx'  -- 👈 替换为你的数据库名
ORDER BY TABLE_ROWS DESC

CREATE TABLE `import_collection_plan` (
                                          `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键ID',
                                          `process_time` char(19) COLLATE utf8_bin DEFAULT NULL COMMENT '程序执行时间',
                                          `serial_no` varchar(32) COLLATE utf8_bin DEFAULT NULL COMMENT '序号',
                                          `enterprise_name` varchar(50) COLLATE utf8_bin DEFAULT NULL COMMENT '填表企业',
                                          `project_no` varchar(100) COLLATE utf8_bin DEFAULT NULL COMMENT '项目编号',
                                          `receipt_no` varchar(50) COLLATE utf8_bin DEFAULT NULL COMMENT '借据编号',
                                          `repayment_period_no` varchar(32) COLLATE utf8_bin DEFAULT NULL COMMENT '还款期次',
                                          `plan_repayment_date` char(10) COLLATE utf8_bin DEFAULT NULL COMMENT '计划还款日期',
                                          `plan_repayment_principal_amt` decimal(18,2) DEFAULT NULL COMMENT '计划还款本金',
                                          `plan_repayment_interest_amt` decimal(18,2) DEFAULT NULL COMMENT '计划还款利息',
                                          `plan_repayment_other_amt` decimal(18,2) DEFAULT NULL COMMENT '计划还款其他金额',
                                          `repayment_status_name` varchar(50) COLLATE utf8_bin DEFAULT NULL COMMENT '还款状态',
                                          `last_recover_date` char(10) COLLATE utf8_bin DEFAULT NULL COMMENT '最后回收时间',
                                          `last_recover_name` varchar(50) COLLATE utf8_bin DEFAULT NULL COMMENT '最后回收人',
                                          `report_date` char(10) COLLATE utf8_bin DEFAULT NULL COMMENT '报送时间',
                                          PRIMARY KEY (`id`),
                                          KEY `enterprise_name` (`enterprise_name`,`report_date`,`process_time`)
) ENGINE=FEDERATED DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='导入收款计划表' CONNECTION='mysql://root:root@192.168.110.164:3106/xxx/import_collection_plan';

create table d select * from import_collection_plan
where report_date = '2023-02-21';

set global general_log = ON

-- 输出到文件（默认）
SET GLOBAL log_output = 'FILE';

-- 或输出到 mysql.general_log 表（方便 SQL 查询）
SET GLOBAL log_output = 'TABLE';

INSERT INTO d select * from import_collection_plan
where report_date = '2023-02-18';

-- 在 192.168.110.164 上执行
CREATE TABLE import_collection_plan_20230221 AS
SELECT * FROM import_collection_plan 
WHERE report_date = '2023-02-21';

// ==UserScript==
// @name         虎嗅评论过滤 - 屏蔽评论数超过100页的用户
// @namespace    http://tampermonkey.net/
// @version      1.0.1
// @description  自动检测并屏蔽评论数超过100页的用户评论
// @author       You
// @match        https://www.huxiu.com/moment/*
// @exclude      https://www.huxiu.com/member/*
// @grant        GM_xmlhttpRequest
// @connect      api-web-account.huxiu.com
// ==/UserScript==

(function() {
    'use strict';

    // 配置
    const MAX_PAGES = 100; // 最大允许的评论页数
    const API_URL = 'https://api-web-account.huxiu.com/web/comment/commentList';
    const CHECK_INTERVAL = 2000; // 检查新评论的间隔（毫秒）

    // 存储正在检查中的用户ID，避免重复请求
    const pendingChecks = new Set();

    /**
     * 从评论元素中提取用户ID
     */
    function extractUserId(commentElement) {
        // 尝试多种方式提取用户ID
        // 方式1: 从虎嗅会员链接中提取（最常见，格式：/member/2374684.html）
        const userLinks = commentElement.querySelectorAll('a[href*="/member/"]');
        for (const link of userLinks) {
            const href = link.getAttribute('href');
            // 匹配 /member/123456.html 或 /member/123456 格式
            let match = href.match(/\/member\/(\d+)(?:\.html)?/);
            if (match) return match[1];
        }

        // 方式2: 从其他用户链接格式中提取
        const otherLinks = commentElement.querySelectorAll('a[href*="/user/"], a[href*="uid="]');
        for (const link of otherLinks) {
            const href = link.getAttribute('href');
            // 匹配 /user/123456 格式
            let match = href.match(/\/user\/(\d+)/);
            if (match) return match[1];
            // 匹配 ?uid=123456 格式
            match = href.match(/[?&]uid=(\d+)/);
            if (match) return match[1];
        }

        // 方式2: 从data属性中提取
        let element = commentElement;
        for (let i = 0; i < 10 && element; i++) {
            const dataUid = element.getAttribute('data-uid') || 
                           element.getAttribute('data-user-id') ||
                           element.getAttribute('uid');
            if (dataUid && /^\d+$/.test(dataUid)) {
                return dataUid;
            }
            element = element.parentElement;
        }

        // 方式3: 从class或id中提取
        element = commentElement;
        for (let i = 0; i < 5 && element; i++) {
            const uidMatch = element.className?.match(/uid[_-]?(\d+)|user[_-]?(\d+)/i) ||
                            element.id?.match(/uid[_-]?(\d+)|user[_-]?(\d+)/i);
            if (uidMatch) {
                return uidMatch[1] || uidMatch[2];
            }
            element = element.parentElement;
        }

        // 方式4: 从图片src或其他属性中提取
        const img = commentElement.querySelector('img[src*="user"], img[src*="avatar"]');
        if (img) {
            const src = img.getAttribute('src');
            const match = src?.match(/user[\/_-]?(\d+)/i);
            if (match) return match[1];
        }

        // 调试：输出元素信息
        console.warn('无法提取用户ID，元素信息:', {
            className: commentElement.className,
            id: commentElement.id,
            innerHTML: commentElement.innerHTML.substring(0, 200)
        });

        return null;
    }

    /**
     * 获取用户评论总数
     */
    function getUserCommentPages(uid) {
        return new Promise((resolve, reject) => {
            // 如果正在检查中，等待
            if (pendingChecks.has(uid)) {
                setTimeout(() => {
                    getUserCommentPages(uid).then(resolve).catch(reject);
                }, 500);
                return;
            }

            pendingChecks.add(uid);

            GM_xmlhttpRequest({
                method: 'POST',
                url: API_URL,
                headers: {
                    'Content-Type': 'application/x-www-form-urlencoded',
                    'Accept': 'application/json',
                    'Referer': 'https://www.huxiu.com/',
                    'Origin': 'https://www.huxiu.com'
                },
                data: `platform=www&page=1&uid=${uid}`,
                onload: function(response) {
                    pendingChecks.delete(uid);
                    try {
                        const data = JSON.parse(response.responseText);
                        
                        // 调试：输出API响应结构
                        console.log(`[API] 用户 ${uid} 的API响应:`, JSON.stringify(data, null, 2));
                        
                        // 尝试多种可能的响应格式
                        let totalPages = 0;
                        
                        if (data && data.data) {
                            // 格式1: 虎嗅API标准格式 { data: { total_page: xxx } }
                            if (data.data.total_page !== undefined && data.data.total_page !== null) {
                                totalPages = parseInt(data.data.total_page);
                                console.log(`[API] 从 data.data.total_page 获取页数: ${totalPages}`);
                            } else {
                                console.warn(`[API] 用户 ${uid} 的响应中未找到 total_page 字段，data.data 内容:`, data.data);
                            }
                        } else {
                            console.warn(`[API] 用户 ${uid} 的响应格式异常，data 或 data.data 不存在:`, data);
                        }
                        
                        if (totalPages === 0) {
                            console.warn(`[API] 用户 ${uid} 的页数解析为0，可能解析失败`);
                        }

                        resolve(totalPages);
                    } catch (e) {
                        console.error('解析API响应失败:', e, response.responseText);
                        reject(e);
                    }
                },
                onerror: function(error) {
                    pendingChecks.delete(uid);
                    console.error('API请求失败:', error);
                    reject(error);
                }
            });
        });
    }

    /**
     * 隐藏评论元素
     * @param {HTMLElement} commentElement - 评论元素（单个评论项）
     * @param {string} uid - 用户ID
     * @param {number} totalPages - 总评论页数
     */
    function hideComment(commentElement, uid, totalPages) {
        // 确保只隐藏单个评论项，而不是整个列表
        // 检查是否是评论列表容器
        if (commentElement.classList.contains('moment-comment__list')) {
            console.warn(`警告：尝试隐藏评论列表容器，跳过。用户ID: ${uid}`);
            return;
        }
        
        // 只隐藏单个评论项
        commentElement.style.display = 'none';
        commentElement.setAttribute('data-filtered', 'true');
        
        // 在控制台输出屏蔽信息
        console.log(`🚫 已屏蔽用户评论 | 用户ID: ${uid} | 总评论页数: ${totalPages}页`);
        
        // 添加一个简单的提示标记，显示评论已被隐藏
        const marker = document.createElement('div');
        marker.style.cssText = 'padding: 8px 12px; background: #f5f5f5; color: #999; font-size: 12px; margin-bottom: 10px; border-left: 3px solid #ddd; border-radius: 2px;';
        marker.textContent = '该评论已隐藏';
        marker.setAttribute('data-filter-marker', 'true');
        
        // 插入到评论项的父容器中，替换被隐藏的评论项位置
        if (commentElement.parentNode) {
            commentElement.parentNode.insertBefore(marker, commentElement);
        }
    }

    /**
     * 检查并过滤单个评论
     */
    async function checkAndFilterComment(commentElement) {
        // 如果已经处理过，跳过
        if (commentElement.getAttribute('data-checked') === 'true' ||
            commentElement.getAttribute('data-filtered') === 'true') {
            return;
        }

        // 安全检查：确保是单个评论项，而不是评论列表容器
        if (commentElement.classList.contains('moment-comment__list')) {
            console.warn('跳过评论列表容器，只处理单个评论项');
            return;
        }

        const uid = extractUserId(commentElement);
        if (!uid) {
            console.warn('无法提取用户ID:', commentElement);
            return;
        }

        // 标记为已检查
        commentElement.setAttribute('data-checked', 'true');

        try {
            const totalPages = await getUserCommentPages(uid);
            console.log(`用户 ${uid} 的评论页数: ${totalPages}`);

            if (totalPages > MAX_PAGES) {
                hideComment(commentElement, uid, totalPages);
            }
        } catch (error) {
            console.error(`检查用户 ${uid} 失败:`, error);
            // 出错时不隐藏，避免误杀
        }
    }

    /**
     * 查找页面上的所有评论元素
     */
    function findAllComments() {
        // 根据虎嗅网站的实际结构，只选择单个评论项
        // 优先使用最精确的选择器，避免选择到评论列表容器
        const selectors = [
            '.comment-item', // 虎嗅单个评论项的标准选择器
            '[data-comment-id]', // 通过data-comment-id属性的单个评论项
        ];

        const comments = new Set();
        
        for (const selector of selectors) {
            try {
                const elements = document.querySelectorAll(selector);
                elements.forEach(el => {
                    // 确保不是已经过滤的元素，且有实际内容
                    // 排除评论列表容器（.moment-comment__list）
                    if (el.getAttribute('data-filtered') !== 'true' &&
                        !el.classList.contains('moment-comment__list') && // 排除列表容器
                        el.offsetHeight > 0 && // 确保元素可见
                        el.textContent.trim().length > 0) { // 确保有内容
                        comments.add(el);
                    }
                });
            } catch (e) {
                // 忽略无效选择器
            }
        }

        // 去重：如果元素A包含元素B，只保留最内层的元素（单个评论项）
        const filtered = Array.from(comments).filter(comment => {
            // 如果这个元素包含其他评论元素，说明它是容器，应该排除
            const hasChildComment = Array.from(comments).some(other => 
                other !== comment && comment.contains(other)
            );
            // 如果这个元素被其他评论元素包含，保留它（它是单个评论项）
            const isChildOfComment = Array.from(comments).some(other => 
                other !== comment && other.contains(comment)
            );
            // 保留：要么是单个评论项（被其他元素包含），要么是独立的评论项（不包含其他评论）
            return !hasChildComment || isChildOfComment;
        });

        return filtered;
    }

    /**
     * 批量检查评论
     */
    async function checkAllComments() {
        const comments = findAllComments();
        console.log(`找到 ${comments.length} 条评论，开始检查...`);

        // 批量处理，避免同时发起太多请求
        const batchSize = 5;
        for (let i = 0; i < comments.length; i += batchSize) {
            const batch = comments.slice(i, i + batchSize);
            await Promise.all(batch.map(comment => checkAndFilterComment(comment)));
            
            // 批次之间稍作延迟
            if (i + batchSize < comments.length) {
                await new Promise(resolve => setTimeout(resolve, 500));
            }
        }
    }

    /**
     * 监听DOM变化，处理动态加载的评论
     */
    function setupMutationObserver() {
        const observer = new MutationObserver((mutations) => {
            let shouldCheck = false;
            
            mutations.forEach((mutation) => {
                mutation.addedNodes.forEach((node) => {
                    if (node.nodeType === 1) { // Element node
                        // 检查是否是评论相关的元素
                        if (node.classList && (
                            node.classList.toString().includes('comment') ||
                            node.querySelector && node.querySelector('[class*="comment"]')
                        )) {
                            shouldCheck = true;
                        }
                    }
                });
            });

            if (shouldCheck) {
                // 延迟检查，等待DOM完全渲染
                setTimeout(() => {
                    checkAllComments();
                }, 1000);
            }
        });

        observer.observe(document.body, {
            childList: true,
            subtree: true
        });
    }

    /**
     * 初始化
     */
    function init() {
        // 排除个人中心页面
        if (window.location.pathname.match(/^\/member\//)) {
            console.log('虎嗅评论过滤插件：跳过个人中心页面');
            return;
        }

        console.log('虎嗅评论过滤插件已启动（无缓存模式）');
        
        // 等待页面加载完成
        if (document.readyState === 'loading') {
            document.addEventListener('DOMContentLoaded', () => {
                setTimeout(checkAllComments, 2000);
                setupMutationObserver();
            });
        } else {
            setTimeout(checkAllComments, 2000);
            setupMutationObserver();
        }

        // 定期检查新评论
        setInterval(checkAllComments, CHECK_INTERVAL);
    }

    // 启动
    init();
})();

SELECT
    table_schema AS database_name,
    table_name,
    table_rows AS approx_rows
FROM information_schema.tables
WHERE table_schema = 'xx'   -- 替换为你的数据库名
  AND engine = 'InnoDB'            -- 可选：只查 InnoDB 表
ORDER BY table_rows DESC;

CREATE DATABASE mysql_db ENGINE = MySQL('localhost:3306', 'test', 'my_user', 'user_password') SETTINGS read_write_timeout=10000, connect_timeout=100;

set allow_experimental_database_materialized_mysql = 1;
CREATE DATABASE tableXXX on cluster 'all-nodes' ENGINE = MaterializedMySQL(
'mysql:3306', 'dbxxxx', 'username', 'pass')
settings 
materialized_mysql_tables_list = 'interface_access_log,enterprise_info'
TABLE OVERRIDE interface_access_log (
    PARTITION BY  toYYYYMM(gmt_create)
    ORDER BY (gmt_create, id)
)

mysqldump \
  --single-transaction \
  --master-data=2 \
  --routines \
  --triggers \
  --events \
  --hex-blob \
  --default-character-set=utf8mb4 \
  --host=127.0.0.1 \
  --port=3306 \
  --user=backup_user \
  --password='your_password' \
  your_database_name \
  > /backup/your_database_$(date +%Y%m%d).sql

主控制连接获取全局读锁（FTWRL）
    让所有表进入“读锁”状态：阻止新的写入，并等待正在进行的写入结束（达到一个全库静止点）。
    这一刻可以认为数据库处于一个确定的、可描述的时间点。

记录复制/增量所需的位置点（metadata）
在读锁还持有时，mydumper 会读取并写出 metadata（常见包含）：
    binlog file/position（以及可能的 GTID） 这保证“这份 dump 对应主库的哪个位置点”是准确的。

所有 worker 线程在屏障(barrier)下建立“同一时间点快照”
    每个线程通常用独立连接去读各自负责的表/分片。
    在 FTWRL 仍然持有时，mydumper 会让这些连接几乎同时执行一致性读相关设置并开启事务快照（典型是 REPEATABLE READ + 一致性快照 语义）。
    因为此时写入被阻塞，所以这些事务拿到的 read view 等价于同一个时间点。

释放 FTWRL，全库恢复可写；dump 线程继续并发读取
    锁释放后，业务写入可以继续。
    但每个线程都在自己的事务快照里读数据：
    InnoDB 的 MVCC 保证它们看到的仍是“锁释放那一刻”的版本（后续提交的新版本对这些事务不可见）。

Java 伪代码：FTWRL + 多线程一致性快照（barrier 同步）

// Pseudo-code (Java-like), illustrating mydumper's idea:
// 1) Hold FTWRL briefly to freeze writes
// 2) Record binlog/gtid position under lock
// 3) Let all worker connections START consistent snapshots at the same point
// 4) Release FTWRL, workers dump concurrently using their own snapshot
 
class DumpCoordinator {
 
  String host;
  int port;
  String user;
  String password;
  String database;
  int threads;
  long chunkSizeBytes;
 
  void runDump() throws Exception {
    Connection ctrl = openConnection();   // control connection
    ctrl.setAutoCommit(true);
 
    // Barrier to ensure all workers have created snapshot before unlocking
    CyclicBarrier snapshotBarrier = new CyclicBarrier(threads + 1);
 
    ExecutorService pool = Executors.newFixedThreadPool(threads);
    List<TableTask> tasks = planTableAndChunkTasks(database, chunkSizeBytes);
 
    // Start workers first (they will wait until coordinator says "snapshot now")
    for (int i = 0; i < threads; i++) {
      pool.submit(new DumpWorker(i, snapshotBarrier, tasks));
    }
 
    // 1) Acquire global read lock (FTWRL)
    exec(ctrl, "FLUSH TABLES WITH READ LOCK");
 
    // 2) Read metadata (binlog pos / gtid) while lock is held
    BinlogPoint p = readBinlogPoint(ctrl);     // e.g. SHOW MASTER STATUS / SHOW BINARY LOG STATUS
    writeMetadataFile(p);
 
    // 3) Tell workers to create consistent snapshot NOW (while FTWRL is still held)
    // coordinator arrives at barrier; workers also arrive after START TRANSACTION WITH CONSISTENT SNAPSHOT
    snapshotBarrier.await(); // releases only when all workers + coordinator reach it
 
    // 4) Release lock quickly so production can continue
    exec(ctrl, "UNLOCK TABLES");
 
    pool.shutdown();
    pool.awaitTermination(24, TimeUnit.HOURS);
    ctrl.close();
  }
 
  Connection openConnection() { /* DriverManager.getConnection(...) */ return null; }
 
  void exec(Connection c, String sql) { /* execute sql */ }
 
  BinlogPoint readBinlogPoint(Connection c) { /* query master status */ return null; }
 
  void writeMetadataFile(BinlogPoint p) { /* write metadata */ }
 
  List<TableTask> planTableAndChunkTasks(String db, long chunkBytes) {
    // Inspect table sizes / PK ranges, split into chunks:
    // - small tables: one task
    // - large tables: multiple chunk tasks (pk ranges)
    return new ArrayList<>();
  }
}
 
class DumpWorker implements Runnable {
  int workerId;
  CyclicBarrier snapshotBarrier;
  List<TableTask> sharedTasks;
 
  DumpWorker(int workerId, CyclicBarrier barrier, List<TableTask> tasks) {
    this.workerId = workerId;
    this.snapshotBarrier = barrier;
    this.sharedTasks = tasks;
  }
 
  @Override
  public void run() {
    Connection conn = openConnection();
    conn.setAutoCommit(false);
 
    // Ensure snapshot semantics (InnoDB MVCC)
    exec(conn, "SET SESSION TRANSACTION ISOLATION LEVEL REPEATABLE READ");
 
    // Create a consistent snapshot while FTWRL is held
    exec(conn, "START TRANSACTION WITH CONSISTENT SNAPSHOT");
 
    // Signal "snapshot ready"
    await(snapshotBarrier);
 
    // After this point coordinator may UNLOCK TABLES;
    // this worker keeps reading the SAME snapshot via MVCC.
    while (true) {
      TableTask task = pollNextTask(sharedTasks);
      if (task == null) break;
 
      dumpTableOrChunk(conn, task);
    }
 
    exec(conn, "COMMIT");
    closeQuietly(conn);
  }
 
  Connection openConnection() { return null; }
 
  void exec(Connection c, String sql) { /* execute sql */ }
 
  void await(CyclicBarrier b) { /* b.await() */ }
 
  TableTask pollNextTask(List<TableTask> tasks) {
    // synchronized(tasks) { pop next }
    return null;
  }
 
  void dumpTableOrChunk(Connection conn, TableTask task) {
    // Example chunk query:
    // SELECT * FROM db.table WHERE pk >= ? AND pk < ? ORDER BY pk;
    // Stream rows -> write file part
  }
 
  void closeQuietly(Connection c) {}
}
 
class TableTask {
  String table;
  boolean isChunk;
  long pkStartInclusive;
  long pkEndExclusive;
}
 
class BinlogPoint {
  String binlogFile;
  long binlogPos;
  String gtidSet; // optional
}

--compress 可以压缩生成zst文件
mydumper --sync-thread-lock-mode=FTWRL --port=3306 --host=192.168.102. --user=root --password='xx' --database=xx --outputdir=/root/testdump --threads=8 --chunk-filesize=128 --verbose=3

# Started dump at: 2026-01-09 14:17:57
[config]
quote-character = BACKTICK

[myloader_session_variables]
SQL_MODE='NO_AUTO_VALUE_ON_ZERO,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION' /*!40101

[source]
# Channel_Name = '' # It can be use to setup replication FOR CHANNEL
# executed_gtid_set = "0-1099-53297657"
# SOURCE_LOG_FILE = "master-bin.000080"
# SOURCE_LOG_POS = 761443754

[`dbxxx`.`admin_punish_credit`]
real_table_name=admin_punish_credit
rows = 26

[`dbxxx`.`admin_punish`]
real_table_name=admin_punish
rows = 23
....

[`dbxxx`.`mydumper_7`]
real_table_name=费用信息
rows = 10000
[config]
max-statement-size = 999998
num-sequences = 0
# Finished dump at: 2026-01-09 14:18:05

-- 假设有一个用户表，包含 50 个字段
CREATE TABLE users (
    id INT PRIMARY KEY,
    username VARCHAR(50),
    email VARCHAR(100),
    -- ... 其他 47 个字段
    last_login TIMESTAMP
);

-- 只更新一个字段
UPDATE users SET last_login = NOW() WHERE id = 1;

-- 监控表膨胀情况
SELECT 
    schemaname,
    tablename,
    pg_size_pretty(pg_total_relation_size(schemaname||'.'||tablename)) as total_size,
    pg_size_pretty(pg_relation_size(schemaname||'.'||tablename)) as table_size,
    pg_size_pretty(pg_total_relation_size(schemaname||'.'||tablename) - pg_relation_size(schemaname||'.'||tablename)) as index_size,
    n_dead_tup,
    n_live_tup,
    round(n_dead_tup * 100.0 / (n_live_tup + n_dead_tup), 2) as dead_ratio
FROM pg_stat_user_tables 
WHERE n_dead_tup > 0
ORDER BY dead_ratio DESC;

-- MySQL 支持多种存储引擎
CREATE TABLE table1 (id INT) ENGINE=InnoDB;    -- 事务支持
CREATE TABLE table2 (id INT) ENGINE=MyISAM;    -- 非事务
CREATE TABLE table3 (id INT) ENGINE=Memory;    -- 内存表

-- PostgreSQL 只有一种存储引擎，但支持多种访问方法
CREATE TABLE table1 (id INT);  -- 默认堆表
CREATE TABLE table2 (id INT) USING heap;  -- 显式指定堆表

-- 支持自定义访问方法（通过扩展）
CREATE EXTENSION zheap;  -- 实验性的新存储引擎
CREATE TABLE table3 (id INT) USING zheap;

-- MySQL 相对简单的数据类型
CREATE TABLE mysql_example (
    id INT AUTO_INCREMENT PRIMARY KEY,
    name VARCHAR(255),
    email VARCHAR(255),
    age TINYINT,
    salary DECIMAL(10,2),
    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
    data JSON
);

-- PostgreSQL 支持更丰富的数据类型
CREATE TABLE postgres_example (
    id SERIAL PRIMARY KEY,  -- 自增序列
    name VARCHAR(255),
    email VARCHAR(255),
    age SMALLINT,  -- 更精确的整数类型
    salary NUMERIC(10,2),  -- 精确数值
    created_at TIMESTAMPTZ DEFAULT NOW(),  -- 带时区的时间戳
    data JSONB,  -- 二进制JSON，支持索引
    tags TEXT[],  -- 数组类型
    status user_status,  -- 枚举类型
    location POINT,  -- 几何类型
    search_vector TSVECTOR,  -- 全文搜索向量
    valid_period DATERANGE  -- 范围类型
);

-- 创建枚举类型
CREATE TYPE user_status AS ENUM ('active', 'inactive', 'pending');

-- 创建测试表
CREATE TABLE test_mvcc (
    id SERIAL PRIMARY KEY,
    name VARCHAR(50),
    value INTEGER
);

-- 插入初始数据
INSERT INTO test_mvcc (name, value) VALUES ('test', 100);

-- 查看元组信息
SELECT ctid, xmin, xmax, * FROM test_mvcc;
-- ctid: 物理位置 (页面号, 行号)
-- xmin: 创建此版本的事务ID
-- xmax: 删除此版本的事务ID (0表示未删除)

-- 开始事务
BEGIN;

-- 更新操作
UPDATE test_mvcc SET value = 200 WHERE id = 1;

-- 在另一个会话中查看
SELECT ctid, xmin, xmax, * FROM test_mvcc;
-- 会看到新的 ctid，说明数据被复制到新位置

COMMIT;

-- 模拟多次更新
BEGIN;
UPDATE test_mvcc SET value = 300 WHERE id = 1;
UPDATE test_mvcc SET value = 400 WHERE id = 1;
COMMIT;

-- 查看版本链（需要特殊工具或扩展）
-- 正常情况下只能看到最新版本

-- 创建一个包含很多字段的表
CREATE TABLE large_table (
    id SERIAL PRIMARY KEY,
    field1 VARCHAR(100),
    field2 VARCHAR(100),
    field3 VARCHAR(100),
    -- ... 假设有 100 个字段
    field100 VARCHAR(100),
    status VARCHAR(20)
);

-- 只更新一个字段
UPDATE large_table SET status = 'active' WHERE id = 1;
-- PostgreSQL 会复制所有 100 个字段到新位置

-- 监控表大小变化
SELECT 
    schemaname,
    tablename,
    pg_size_pretty(pg_total_relation_size(schemaname||'.'||tablename)) as size,
    n_tup_ins as inserts,
    n_tup_upd as updates,
    n_tup_del as deletes
FROM pg_stat_user_tables 
WHERE tablename = 'large_table';

-- 安装 zheap 扩展（需要编译支持）
CREATE EXTENSION zheap;

-- 使用 zheap 存储引擎
CREATE TABLE optimized_table (
    id SERIAL PRIMARY KEY,
    data TEXT,
    status VARCHAR(20)
) USING zheap;

-- 避免过宽的表，考虑垂直分表
CREATE TABLE user_basic_info (
    id SERIAL PRIMARY KEY,
    username VARCHAR(50),
    email VARCHAR(100)
);

CREATE TABLE user_extended_info (
    user_id INTEGER REFERENCES user_basic_info(id),
    profile_data JSONB,
    preferences JSONB
);

# 安装 pg_repack
# Ubuntu/Debian
sudo apt-get install postgresql-15-repack

# 重组表
pg_repack -d your_database -t your_table

-- 创建监控视图
CREATE OR REPLACE VIEW table_bloat_monitor AS
SELECT 
    schemaname,
    tablename,
    pg_size_pretty(pg_total_relation_size(schemaname||'.'||tablename)) as total_size,
    pg_size_pretty(pg_relation_size(schemaname||'.'||tablename)) as table_size,
    n_live_tup as live_tuples,
    n_dead_tup as dead_tuples,
    CASE 
        WHEN n_live_tup + n_dead_tup > 0 
        THEN round(n_dead_tup * 100.0 / (n_live_tup + n_dead_tup), 2)
        ELSE 0 
    END as dead_ratio,
    last_vacuum,
    last_autovacuum,
    last_analyze,
    last_autoanalyze
FROM pg_stat_user_tables 
WHERE n_dead_tup > 0
ORDER BY dead_ratio DESC;

-- 使用监控视图
SELECT * FROM table_bloat_monitor WHERE dead_ratio > 10;

-- 全局配置
ALTER SYSTEM SET autovacuum = on;
ALTER SYSTEM SET autovacuum_max_workers = 3;
ALTER SYSTEM SET autovacuum_naptime = '1min';
ALTER SYSTEM SET autovacuum_vacuum_threshold = 50;
ALTER SYSTEM SET autovacuum_analyze_threshold = 50;
ALTER SYSTEM SET autovacuum_vacuum_scale_factor = 0.1;  -- 降低到10%
ALTER SYSTEM SET autovacuum_analyze_scale_factor = 0.05;  -- 降低到5%

-- 表级配置（针对大表）
ALTER TABLE large_table SET (
    autovacuum_vacuum_scale_factor = 0.05,  -- 5% 触发
    autovacuum_analyze_scale_factor = 0.02,  -- 2% 触发
    autovacuum_vacuum_cost_delay = 10,  -- 降低延迟
    autovacuum_vacuum_cost_limit = 1000  -- 增加限制
);

-- 重载配置
SELECT pg_reload_conf();

-- 普通 vacuum（不阻塞读写）
VACUUM ANALYZE your_table;

-- 完整 vacuum（阻塞写入，回收空间）
VACUUM FULL your_table;

-- 使用 pg_repack（在线重组，不阻塞）
-- pg_repack -d your_database -t your_table

-- 创建支持 HOT 的表结构
CREATE TABLE hot_optimized (
    id SERIAL PRIMARY KEY,
    name VARCHAR(50),
    email VARCHAR(100),
    status VARCHAR(20),
    created_at TIMESTAMPTZ DEFAULT NOW()
);

-- 创建索引（只对需要查询的字段）
CREATE INDEX idx_hot_optimized_name ON hot_optimized(name);
CREATE INDEX idx_hot_optimized_status ON hot_optimized(status);

-- 更新不涉及索引字段的列（HOT 更新）
UPDATE hot_optimized SET email = 'new@example.com' WHERE id = 1;
-- 这个更新可能使用 HOT，因为 email 字段没有索引

-- 查看 HOT 更新统计
SELECT 
    schemaname,
    tablename,
    n_tup_hot_upd as hot_updates,
    n_tup_upd as total_updates,
    CASE 
        WHEN n_tup_upd > 0 
        THEN round(n_tup_hot_upd * 100.0 / n_tup_upd, 2)
        ELSE 0 
    END as hot_ratio
FROM pg_stat_user_tables 
WHERE n_tup_upd > 0
ORDER BY hot_ratio DESC;

-- 避免在频繁更新的字段上创建索引
-- 错误示例：在状态字段上创建索引，但状态经常变化
CREATE INDEX idx_bad_status ON orders(status);  -- 避免

-- 正确示例：在相对稳定的字段上创建索引
CREATE INDEX idx_good_customer ON orders(customer_id);  -- 推荐

-- 使用部分索引
CREATE INDEX idx_active_orders ON orders(customer_id) 
WHERE status = 'active';

-- 创建 Vacuum 监控视图
CREATE OR REPLACE VIEW vacuum_monitor AS
SELECT 
    schemaname,
    tablename,
    last_vacuum,
    last_autovacuum,
    last_analyze,
    last_autoanalyze,
    vacuum_count,
    autovacuum_count,
    analyze_count,
    autoanalyze_count,
    CASE 
        WHEN last_autovacuum IS NULL THEN 'Never'
        WHEN last_autovacuum < NOW() - INTERVAL '1 day' THEN 'Stale'
        ELSE 'Recent'
    END as vacuum_status
FROM pg_stat_user_tables
ORDER BY last_autovacuum NULLS FIRST;

-- 使用监控视图
SELECT * FROM vacuum_monitor WHERE vacuum_status IN ('Never', 'Stale');

-- 查看长时间运行的事务
SELECT 
    pid,
    usename,
    application_name,
    client_addr,
    state,
    query_start,
    now() - query_start as duration,
    query
FROM pg_stat_activity 
WHERE state IN ('active', 'idle in transaction')
  AND now() - query_start > INTERVAL '1 hour'
ORDER BY duration DESC;

-- 查看 Vacuum 进程
SELECT 
    pid,
    usename,
    application_name,
    state,
    query_start,
    query
FROM pg_stat_activity 
WHERE query LIKE '%VACUUM%' OR query LIKE '%ANALYZE%';

-- 针对不同表设置不同的 Vacuum 策略
-- 大表：更频繁的 Vacuum
ALTER TABLE large_frequently_updated_table SET (
    autovacuum_vacuum_scale_factor = 0.02,  -- 2%
    autovacuum_analyze_scale_factor = 0.01,  -- 1%
    autovacuum_vacuum_cost_delay = 5,  -- 更积极的 Vacuum
    autovacuum_vacuum_cost_limit = 2000
);

-- 小表：标准设置
ALTER TABLE small_stable_table SET (
    autovacuum_vacuum_scale_factor = 0.2,  -- 20%
    autovacuum_analyze_scale_factor = 0.1   -- 10%
);

-- 只读表：禁用 Autovacuum
ALTER TABLE read_only_table SET (
    autovacuum_enabled = false
);

# 添加 PostgreSQL 官方仓库
sudo apt update
sudo apt install -y wget ca-certificates
wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list

# 安装 PostgreSQL 15
sudo apt update
sudo apt install -y postgresql-15 postgresql-client-15 postgresql-contrib-15

# 启动服务
sudo systemctl start postgresql
sudo systemctl enable postgresql

# 安装 PostgreSQL 官方仓库
sudo yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm

# 安装 PostgreSQL 15
sudo yum install -y postgresql15-server postgresql15 postgresql15-contrib

# 初始化数据库
sudo /usr/pgsql-15/bin/postgresql-15-setup initdb

# 启动服务
sudo systemctl start postgresql-15
sudo systemctl enable postgresql-15

# 使用 Docker 运行 PostgreSQL
docker run --name postgres-15 \
  -e POSTGRES_PASSWORD=your_password \
  -e POSTGRES_DB=your_database \
  -p 5432:5432 \
  -v postgres_data:/var/lib/postgresql/data \
  -d postgres:15

# 连接到容器
docker exec -it postgres-15 psql -U postgres

# 编辑 postgresql.conf
sudo nano /etc/postgresql/15/main/postgresql.conf

# 关键配置项
listen_addresses = '*'          # 允许外部连接
port = 5432                     # 端口号
max_connections = 100           # 最大连接数
shared_buffers = 256MB          # 共享缓冲区
effective_cache_size = 1GB      # 有效缓存大小
work_mem = 4MB                  # 工作内存
maintenance_work_mem = 64MB     # 维护工作内存

# 编辑 pg_hba.conf
sudo nano /etc/postgresql/15/main/pg_hba.conf

# 添加连接规则
# TYPE  DATABASE        USER            ADDRESS                 METHOD
local   all             postgres                                peer
local   all             all                                     md5
host    all             all             127.0.0.1/32            md5
host    all             all             ::1/128                 md5
host    all             all             0.0.0.0/0               md5

# 重启 PostgreSQL 服务
sudo systemctl restart postgresql

# 检查服务状态
sudo systemctl status postgresql

# 查看日志
sudo journalctl -u postgresql -f

-- 连接到 PostgreSQL
sudo -u postgres psql

-- 创建用户
CREATE USER app_user WITH PASSWORD 'secure_password';

-- 创建数据库
CREATE DATABASE app_database OWNER app_user;

-- 授予权限
GRANT ALL PRIVILEGES ON DATABASE app_database TO app_user;

-- 连接到新数据库
\c app_database

-- 授予模式权限
GRANT ALL ON SCHEMA public TO app_user;
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO app_user;
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO app_user;

-- 设置默认权限
ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO app_user;
ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO app_user;

-- 创建角色
CREATE ROLE readonly_role;
CREATE ROLE write_role;

-- 授予权限
GRANT CONNECT ON DATABASE app_database TO readonly_role;
GRANT USAGE ON SCHEMA public TO readonly_role;
GRANT SELECT ON ALL TABLES IN SCHEMA public TO readonly_role;

GRANT CONNECT ON DATABASE app_database TO write_role;
GRANT USAGE ON SCHEMA public TO write_role;
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO write_role;

-- 将用户添加到角色
GRANT readonly_role TO app_user;
GRANT write_role TO app_user;

-- MySQL 表结构
CREATE TABLE mysql_numeric (
    id TINYINT AUTO_INCREMENT PRIMARY KEY,
    small_num SMALLINT,
    medium_num MEDIUMINT,
    normal_num INT,
    big_num BIGINT,
    decimal_num DECIMAL(10,2),
    float_num FLOAT,
    double_num DOUBLE
);

-- PostgreSQL 对应表结构
CREATE TABLE postgres_numeric (
    id SMALLSERIAL PRIMARY KEY,  -- 自增小整数
    small_num SMALLINT,
    medium_num INTEGER,          -- MEDIUMINT 映射到 INTEGER
    normal_num INTEGER,
    big_num BIGINT,
    decimal_num NUMERIC(10,2),   -- DECIMAL 改为 NUMERIC
    float_num REAL,              -- FLOAT 改为 REAL
    double_num DOUBLE PRECISION  -- DOUBLE 改为 DOUBLE PRECISION
);

-- MySQL 字符串表
CREATE TABLE mysql_strings (
    id INT AUTO_INCREMENT PRIMARY KEY,
    fixed_char CHAR(10),
    variable_char VARCHAR(255),
    long_text LONGTEXT,
    status ENUM('active', 'inactive', 'pending')
);

-- PostgreSQL 对应表结构
CREATE TYPE user_status AS ENUM ('active', 'inactive', 'pending');

CREATE TABLE postgres_strings (
    id SERIAL PRIMARY KEY,
    fixed_char CHAR(10),
    variable_char VARCHAR(255),
    long_text TEXT,                    -- 所有文本类型统一为 TEXT
    status user_status                 -- 使用自定义枚举类型
);

-- 或者使用 CHECK 约束
CREATE TABLE postgres_strings_check (
    id SERIAL PRIMARY KEY,
    fixed_char CHAR(10),
    variable_char VARCHAR(255),
    long_text TEXT,
    status VARCHAR(20) CHECK (status IN ('active', 'inactive', 'pending'))
);

-- MySQL 日期时间表
CREATE TABLE mysql_datetime (
    id INT AUTO_INCREMENT PRIMARY KEY,
    birth_date DATE,
    work_time TIME,
    created_at DATETIME,
    updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
    birth_year YEAR
);

-- PostgreSQL 对应表结构
CREATE TABLE postgres_datetime (
    id SERIAL PRIMARY KEY,
    birth_date DATE,
    work_time TIME,
    created_at TIMESTAMP,                    -- DATETIME 改为 TIMESTAMP
    updated_at TIMESTAMPTZ DEFAULT NOW(),   -- 带时区的时间戳
    birth_year SMALLINT                     -- YEAR 改为 SMALLINT
);

-- 创建自动更新触发器
CREATE OR REPLACE FUNCTION update_updated_at_column()
RETURNS TRIGGER AS $$
BEGIN
    NEW.updated_at = NOW();
    RETURN NEW;
END;
$$ language 'plpgsql';

CREATE TRIGGER update_postgres_datetime_updated_at 
    BEFORE UPDATE ON postgres_datetime 
    FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();

-- MySQL JSON 表
CREATE TABLE mysql_json (
    id INT AUTO_INCREMENT PRIMARY KEY,
    user_data JSON,
    settings JSON
);

-- PostgreSQL JSONB 表
CREATE TABLE postgres_jsonb (
    id SERIAL PRIMARY KEY,
    user_data JSONB,    -- 使用 JSONB 而不是 JSON
    settings JSONB
);

-- 创建 GIN 索引支持 JSON 查询
CREATE INDEX idx_user_data_gin ON postgres_jsonb USING gin (user_data);
CREATE INDEX idx_settings_gin ON postgres_jsonb USING gin (settings);

-- JSON 查询示例
-- MySQL 查询
SELECT * FROM mysql_json WHERE JSON_EXTRACT(user_data, '$.name') = 'John';

-- PostgreSQL 查询
SELECT * FROM postgres_jsonb WHERE user_data->>'name' = 'John';
SELECT * FROM postgres_jsonb WHERE user_data @> '{"status": "active"}';
SELECT * FROM postgres_jsonb WHERE user_data ? 'email';

-- PostgreSQL 支持数组类型（MySQL 不支持）
CREATE TABLE postgres_arrays (
    id SERIAL PRIMARY KEY,
    tags TEXT[],                    -- 文本数组
    scores INTEGER[],               -- 整数数组
    coordinates FLOAT[][],          -- 二维浮点数组
    metadata JSONB[]                -- JSONB 数组
);

-- 插入数组数据
INSERT INTO postgres_arrays (tags, scores, coordinates, metadata) VALUES (
    ARRAY['tag1', 'tag2', 'tag3'],
    ARRAY[85, 92, 78],
    ARRAY[[1.0, 2.0], [3.0, 4.0]],
    ARRAY['{"key": "value1"}', '{"key": "value2"}']
);

-- 数组查询
SELECT * FROM postgres_arrays WHERE 'tag1' = ANY(tags);
SELECT * FROM postgres_arrays WHERE array_length(scores, 1) > 2;
SELECT * FROM postgres_arrays WHERE tags @> ARRAY['tag1'];

-- 创建数组索引
CREATE INDEX idx_tags_gin ON postgres_arrays USING gin (tags);

-- PostgreSQL 支持范围类型（MySQL 不支持）
CREATE TABLE postgres_ranges (
    id SERIAL PRIMARY KEY,
    price_range NUMRANGE,           -- 数值范围
    date_range DATERANGE,           -- 日期范围
    time_range TSRANGE,             -- 时间戳范围
    text_range INTRANGE             -- 整数范围
);

-- 插入范围数据
INSERT INTO postgres_ranges (price_range, date_range, time_range, text_range) VALUES (
    '[100, 500)',                   -- 100 到 500（不包含 500）
    '[2023-01-01, 2023-12-31]',    -- 2023 年全年
    '[2023-01-01 00:00:00, 2023-01-01 23:59:59]',
    '[1, 10]'                       -- 1 到 10
);

-- 范围查询
SELECT * FROM postgres_ranges WHERE price_range @> 250;  -- 包含 250
SELECT * FROM postgres_ranges WHERE date_range && '[2023-06-01, 2023-06-30]';  -- 重叠
SELECT * FROM postgres_ranges WHERE price_range <@ '[0, 1000]';  -- 被包含

-- 创建范围索引
CREATE INDEX idx_price_range ON postgres_ranges USING gist (price_range);

-- 推荐公式
work_mem = (总内存 * 0.8 - shared_buffers) / 活跃连接数

-- 示例：16GB 内存，100 个连接
work_mem = (16GB * 0.8 - 4GB) / 100 = 96MB

-- 查看临时文件使用情况
SELECT 
    datname,
    temp_files,
    temp_bytes,
    pg_size_pretty(temp_bytes) as temp_size
FROM pg_stat_database 
WHERE temp_files > 0
ORDER BY temp_bytes DESC;

-- 查看当前排序操作
SELECT 
    pid,
    usename,
    application_name,
    query,
    state
FROM pg_stat_activity 
WHERE query LIKE '%ORDER BY%' 
   OR query LIKE '%GROUP BY%'
   OR query LIKE '%DISTINCT%';

-- 针对不同规模系统的配置建议

-- 小型系统 (4GB 内存)
shared_buffers = 1GB
effective_cache_size = 3GB
work_mem = 4MB
maintenance_work_mem = 64MB
temp_buffers = 8MB

-- 中型系统 (16GB 内存)
shared_buffers = 4GB
effective_cache_size = 12GB
work_mem = 16MB
maintenance_work_mem = 256MB
temp_buffers = 8MB

-- 大型系统 (64GB 内存)
shared_buffers = 16GB
effective_cache_size = 48GB
work_mem = 64MB
maintenance_work_mem = 1GB
temp_buffers = 8MB

# /etc/pgbouncer/pgbouncer.ini
[databases]
app_db = host=127.0.0.1 port=5432 dbname=app_database pool_size=100

[pgbouncer]
listen_addr = 127.0.0.1
listen_port = 6432
auth_type = md5
auth_file = /etc/pgbouncer/userlist.txt
pool_mode = transaction
max_client_conn = 1000
default_pool_size = 100
reserve_pool_size = 10
reserve_pool_timeout = 5
log_connections = 1
log_disconnections = 1
log_pooler_errors = 1

-- 查看当前连接
SELECT 
    datname,
    usename,
    application_name,
    client_addr,
    state,
    query_start,
    now() - query_start as duration,
    query
FROM pg_stat_activity 
WHERE state != 'idle'
ORDER BY query_start;

-- 查看连接统计
SELECT 
    datname,
    numbackends as current_connections,
    max_connections,
    round(numbackends * 100.0 / max_connections, 2) as connection_usage
FROM pg_stat_database 
JOIN pg_database ON pg_stat_database.datname = pg_database.datname;

-- 可能较慢的 CTE 写法
WITH user_stats AS (
    SELECT user_id, COUNT(*) as order_count
    FROM orders GROUP BY user_id
)
SELECT u.name, us.order_count
FROM users u JOIN user_stats us ON u.id = us.user_id;

-- 通常更快的子查询写法
SELECT u.name, us.order_count
FROM users u JOIN (
    SELECT user_id, COUNT(*) as order_count
    FROM orders GROUP BY user_id
) us ON u.id = us.user_id;

-- PostgreSQL 不会自动为外键创建索引
-- 需要手动创建
CREATE INDEX CONCURRENTLY idx_orders_user_id ON orders(user_id);

-- 复合索引顺序很重要
CREATE INDEX idx_orders_status_created ON orders(status, created_at);
-- 支持 (status), (status, created_at) 查询

# pgbouncer 配置
[databases]
mydb = host=127.0.0.1 port=5432 pool_size=100

[pgbouncer]
pool_mode = transaction  # 事务级连接池
max_client_conn = 1000

-- 避免：复杂的嵌套函数
CREATE OR REPLACE FUNCTION complex_business_logic()
RETURNS TABLE(...) AS $$
BEGIN
    -- 大量内存操作和递归调用
    -- 影响数据库性能
END;
$$ LANGUAGE plpgsql;

-- 每个表最多一个 BEFORE 和一个 AFTER 触发器
CREATE OR REPLACE FUNCTION before_orders()
RETURNS TRIGGER AS $$
BEGIN
    -- 所有逻辑集中在一个函数中
    IF TG_OP = 'INSERT' THEN
        -- 插入逻辑
    ELSIF TG_OP = 'UPDATE' THEN
        -- 更新逻辑
    END IF;
    RETURN COALESCE(NEW, OLD);
END;
$$ LANGUAGE plpgsql;

-- 替代方案：事件队列表
CREATE TABLE event_queue (
    id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
    user_id uuid NOT NULL,
    type text NOT NULL,
    data jsonb NOT NULL,
    created_at timestamptz NOT NULL DEFAULT now(),
    acquired_at timestamptz
);

-- 批量处理事件
UPDATE event_queue 
SET acquired_at = now() 
WHERE id IN (
    SELECT id FROM event_queue 
    WHERE acquired_at IS NULL 
    ORDER BY created_at 
    FOR UPDATE SKIP LOCKED 
    LIMIT 1000
) RETURNING *;

-- 避免：不使用索引
SELECT * FROM users WHERE email IS NOT DISTINCT FROM 'test@example.com';

-- 推荐：显式 NULL 检查
SELECT * FROM users 
WHERE (email IS NULL AND 'test@example.com' IS NULL) 
   OR email = 'test@example.com';

-- 安装关键监控扩展
CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
CREATE EXTENSION IF NOT EXISTS pg_qualstats;
CREATE EXTENSION IF NOT EXISTS pg_wait_sampling;

-- 查看慢查询
SELECT query, total_time, calls, mean_time
FROM pg_stat_statements 
ORDER BY total_time DESC 
LIMIT 10;

-- JSONB 文档存储
CREATE TABLE products (
    id serial PRIMARY KEY,
    metadata jsonb,
    created_at timestamptz DEFAULT now()
);

-- 创建 GIN 索引支持复杂查询
CREATE INDEX idx_products_metadata ON products USING gin (metadata);

-- 查询示例
SELECT * FROM products 
WHERE metadata @> '{"category": "electronics", "price": {"$gt": 500}}';

-- TimescaleDB 超表
SELECT create_hypertable('sensor_data', 'timestamp');

-- 自动分区和压缩
ALTER TABLE sensor_data SET (
    timescaledb.compress,
    timescaledb.compress_orderby = 'timestamp DESC'
);

-- 连接其他 PostgreSQL 实例
CREATE EXTENSION postgres_fdw;

CREATE SERVER remote_server 
FOREIGN DATA WRAPPER postgres_fdw 
OPTIONS (host 'remote-host', port '5432', dbname 'remote_db');

-- 联邦查询
SELECT l.name, r.amount 
FROM local_customers l 
JOIN remote_orders r ON l.id = r.customer_id;

-- 数据库健康检查
SELECT 
    datname,
    numbackends as connections,
    xact_commit + xact_rollback as transactions,
    blks_read + blks_hit as total_blocks,
    round(blks_hit * 100.0 / (blks_hit + blks_read), 2) as cache_hit_ratio
FROM pg_stat_database 
WHERE datname = current_database();

-- 监控表膨胀
SELECT 
    schemaname,
    tablename,
    pg_size_pretty(pg_total_relation_size(schemaname||'.'||tablename)) as size,
    n_dead_tup,
    n_live_tup,
    round(n_dead_tup * 100.0 / (n_live_tup + n_dead_tup), 2) as dead_ratio
FROM pg_stat_user_tables 
WHERE n_dead_tup > 0
ORDER BY dead_ratio DESC;

-- 检查未使用的索引
SELECT 
    schemaname,
    tablename,
    indexname,
    idx_tup_read,
    idx_tup_fetch
FROM pg_stat_user_indexes 
WHERE idx_tup_read = 0 
  AND idx_tup_fetch = 0;

# 使用 pgloader 迁移
pgloader mysql://user:pass@mysql-host/dbname \
         postgresql://user:pass@pg-host/dbname

# 使用 ora2pg 从 Oracle 迁移（也可用于 MySQL）
ora2pg -c config/ora2pg.conf

-- 创建测试环境
CREATE DATABASE test_migration;

-- 运行性能基准
\timing on
EXPLAIN ANALYZE SELECT * FROM large_table WHERE indexed_column = 'value';

-- 对比迁移前后的性能指标

# 检查服务状态
sudo systemctl status postgresql

# 检查端口是否监听
sudo netstat -tlnp | grep 5432

# 检查配置文件
sudo nano /etc/postgresql/15/main/postgresql.conf
# 确保 listen_addresses = '*'

# 检查认证配置
sudo nano /etc/postgresql/15/main/pg_hba.conf
# 确保有正确的连接规则

# 重启服务
sudo systemctl restart postgresql

-- 检查用户是否存在
SELECT usename FROM pg_user WHERE usename = 'your_username';

-- 重置密码
ALTER USER your_username WITH PASSWORD 'new_password';

-- 检查用户权限
\du your_username

-- 启用查询统计
CREATE EXTENSION IF NOT EXISTS pg_stat_statements;

-- 查看慢查询
SELECT 
    query,
    calls,
    total_time,
    mean_time,
    rows,
    100.0 * shared_blks_hit / nullif(shared_blks_hit + shared_blks_read, 0) AS hit_percent
FROM pg_stat_statements 
ORDER BY mean_time DESC 
LIMIT 10;

-- 分析查询计划
EXPLAIN (ANALYZE, BUFFERS) SELECT * FROM your_table WHERE condition;

-- 检查表膨胀
SELECT 
    schemaname,
    tablename,
    n_dead_tup,
    n_live_tup,
    round(n_dead_tup * 100.0 / (n_live_tup + n_dead_tup), 2) as dead_ratio
FROM pg_stat_user_tables 
WHERE n_dead_tup > 0
ORDER BY dead_ratio DESC;

-- 手动执行 vacuum
VACUUM ANALYZE your_table;

-- 如果膨胀严重，使用 pg_repack
-- pg_repack -d your_database -t your_table

-- 查看当前锁
SELECT 
    blocked_locks.pid AS blocked_pid,
    blocked_activity.usename AS blocked_user,
    blocking_locks.pid AS blocking_pid,
    blocking_activity.usename AS blocking_user,
    blocked_activity.query AS blocked_statement,
    blocking_activity.query AS current_statement_in_blocking_process
FROM pg_catalog.pg_locks blocked_locks
JOIN pg_catalog.pg_stat_activity blocked_activity ON blocked_activity.pid = blocked_locks.pid
JOIN pg_catalog.pg_locks blocking_locks ON blocking_locks.locktype = blocked_locks.locktype
    AND blocking_locks.database IS NOT DISTINCT FROM blocked_locks.database
    AND blocking_locks.relation IS NOT DISTINCT FROM blocked_locks.relation
    AND blocking_locks.page IS NOT DISTINCT FROM blocked_locks.page
    AND blocking_locks.tuple IS NOT DISTINCT FROM blocked_locks.tuple
    AND blocking_locks.virtualxid IS NOT DISTINCT FROM blocked_locks.virtualxid
    AND blocking_locks.transactionid IS NOT DISTINCT FROM blocked_locks.transactionid
    AND blocking_locks.classid IS NOT DISTINCT FROM blocked_locks.classid
    AND blocking_locks.objid IS NOT DISTINCT FROM blocked_locks.objid
    AND blocking_locks.objsubid IS NOT DISTINCT FROM blocked_locks.objsubid
    AND blocking_locks.pid != blocked_locks.pid
JOIN pg_catalog.pg_stat_activity blocking_activity ON blocking_activity.pid = blocking_locks.pid
WHERE NOT blocked_locks.granted;

-- 终止阻塞的查询
SELECT pg_terminate_backend(blocked_pid);

-- 检查数据库大小
SELECT 
    datname,
    pg_size_pretty(pg_database_size(datname)) as size
FROM pg_database
ORDER BY pg_database_size(datname) DESC;

-- 检查表大小
SELECT 
    schemaname,
    tablename,
    pg_size_pretty(pg_total_relation_size(schemaname||'.'||tablename)) as size
FROM pg_stat_user_tables
ORDER BY pg_total_relation_size(schemaname||'.'||tablename) DESC;

-- 清理 WAL 日志（谨慎操作）
-- 首先检查 WAL 日志大小
SELECT pg_size_pretty(pg_wal_lsn_diff(pg_current_wal_lsn(), '0/0'));

-- 手动切换 WAL 日志
SELECT pg_switch_wal();

-- 查看当前配置
SELECT name, setting, unit, context, short_desc 
FROM pg_settings 
WHERE name IN ('shared_buffers', 'work_mem', 'effective_cache_size');

-- 修改配置
ALTER SYSTEM SET shared_buffers = '256MB';
SELECT pg_reload_conf();

-- 查看配置是否生效
SHOW shared_buffers;

#!/bin/bash
# postgres_health_check.sh

echo "=== PostgreSQL Health Check ==="
echo "Date: $(date)"
echo

# 检查服务状态
echo "1. Service Status:"
systemctl is-active postgresql

# 检查连接数
echo -e "\n2. Connection Status:"
psql -U postgres -c "
SELECT 
    datname,
    numbackends as current_connections,
    max_connections,
    round(numbackends * 100.0 / max_connections, 2) as usage_percent
FROM pg_stat_database 
JOIN pg_database ON pg_stat_database.datname = pg_database.datname
WHERE datname NOT IN ('template0', 'template1', 'postgres');
"

# 检查表膨胀
echo -e "\n3. Table Bloat Check:"
psql -U postgres -c "
SELECT 
    schemaname,
    tablename,
    n_dead_tup,
    n_live_tup,
    round(n_dead_tup * 100.0 / (n_live_tup + n_dead_tup), 2) as dead_ratio
FROM pg_stat_user_tables 
WHERE n_dead_tup > 0 AND n_live_tup + n_dead_tup > 1000
ORDER BY dead_ratio DESC
LIMIT 10;
"

# 检查慢查询
echo -e "\n4. Slow Queries:"
psql -U postgres -c "
SELECT 
    query,
    calls,
    mean_time,
    total_time
FROM pg_stat_statements 
ORDER BY mean_time DESC 
LIMIT 5;
"

# 检查锁
echo -e "\n5. Lock Status:"
psql -U postgres -c "
SELECT 
    mode,
    count(*) as lock_count
FROM pg_locks 
GROUP BY mode
ORDER BY lock_count DESC;
"

echo -e "\n=== Health Check Complete ==="

#!/bin/bash
# postgres_performance_monitor.sh

LOG_FILE="/var/log/postgres_performance.log"
DATE=$(date '+%Y-%m-%d %H:%M:%S')

echo "[$DATE] Performance Check" >> $LOG_FILE

# 检查缓存命中率
CACHE_HIT=$(psql -U postgres -t -c "
SELECT round(100.0 * sum(blks_hit) / (sum(blks_hit) + sum(blks_read)), 2)
FROM pg_stat_database 
WHERE datname NOT IN ('template0', 'template1', 'postgres');
")

echo "[$DATE] Cache Hit Ratio: $CACHE_HIT%" >> $LOG_FILE

# 检查活跃连接
ACTIVE_CONNECTIONS=$(psql -U postgres -t -c "
SELECT count(*) FROM pg_stat_activity WHERE state = 'active';
")

echo "[$DATE] Active Connections: $ACTIVE_CONNECTIONS" >> $LOG_FILE

# 检查数据库大小
DB_SIZE=$(psql -U postgres -t -c "
SELECT pg_size_pretty(sum(pg_database_size(datname)))
FROM pg_database 
WHERE datname NOT IN ('template0', 'template1', 'postgres');
")

echo "[$DATE] Total Database Size: $DB_SIZE" >> $LOG_FILE

# 检查 WAL 日志大小
WAL_SIZE=$(psql -U postgres -t -c "
SELECT pg_size_pretty(pg_wal_lsn_diff(pg_current_wal_lsn(), '0/0'));
")

echo "[$DATE] WAL Size: $WAL_SIZE" >> $LOG_FILE

echo "[$DATE] Performance Check Complete" >> $LOG_FILE
echo "---" >> $LOG_FILE

# 1. 停止 PostgreSQL 服务
sudo systemctl stop postgresql

# 2. 备份当前数据目录
sudo cp -r /var/lib/postgresql/15/main /var/lib/postgresql/15/main.backup.$(date +%Y%m%d_%H%M%S)

# 3. 从备份恢复
sudo -u postgres pg_restore -d your_database /path/to/backup.dump

# 4. 启动服务
sudo systemctl start postgresql

# 5. 验证数据
psql -U postgres -d your_database -c "SELECT count(*) FROM your_table;"

# 恢复配置文件
sudo cp /etc/postgresql/15/main/postgresql.conf.backup /etc/postgresql/15/main/postgresql.conf
sudo cp /etc/postgresql/15/main/pg_hba.conf.backup /etc/postgresql/15/main/pg_hba.conf

# 重启服务
sudo systemctl restart postgresql

https://geo.datav.aliyun.com/areas_v3/bound/geojson?code=510000_full

create table user(
id int primary key,
province_id int,
city_id int,
name varchar
);

select city_id, count(*) from user
where province_id = 510000 and city_id is not null
GROUP BY  city_id

docker pull edwardg/metabase:v0.54.9.1

FROM metabase/metabase:v0.54.9

# 安装zip工具
USER root
RUN apk update && apk add --no-cache zip

# 创建临时工作目录
WORKDIR /tmp

COPY mariadb-java-client-2.7.10.jar mariadb-java-client-2.7.10.jar

# 使用单层RUN命令进行所有操作
RUN unzip /app/metabase.jar -d metabase-extracted && \
    unzip /tmp/mariadb-java-client-2.7.10.jar -d mariadb-extracted && \
    cp -r mariadb-extracted/org/mariadb/jdbc/Driver* metabase-extracted/org/mariadb/jdbc/ && \
    cp -r mariadb-extracted/org/mariadb/jdbc/ProxyedSqlComponent* metabase-extracted/org/mariadb/jdbc/ && \
    cd metabase-extracted && \
    zip -r /app/metabase.jar ./* && \
    rm -rf /tmp/metabase-extracted /tmp/mariadb-extracted /tmp/mariadb-java-client-2.7.10.jar

# 恢复工作目录
WORKDIR /app

-- 动态排序（添加ORDER BY clause）
SET @group_by_field = ;

-- 拼接动态SQL（包含过滤和排序）
SET @sql = CONCAT(
        'SELECT ',
        CASE @group_by_field
            WHEN 'org_id' THEN '`province_id` AS `省`, `city_id` AS `市`, count(*) AS `count`'
            WHEN 'auth_status' THEN '`auth_status` AS `认证状态`,  count(*) AS `count`'
            END,
        ' FROM `user` ',
    -- 动态过滤
    -- 动态分组
        'GROUP BY ',
        CASE @group_by_field
            WHEN 'org_id' THEN '1,2'
            WHEN 'auth_status' THEN '1'
            END
    );

-- 预处理SQL（将@sql转换为可执行的语句）
PREPARE stmt FROM @sql;

-- 执行预处理语句
EXECUTE stmt