Bootstrap

【Web系列三十】MYSQL库表比对升级脚本

写在前面

        随着软件的迭代开发，数据库表有变动是常有的事，如果没有在开发时记录变更情况的话。对于线上生产环境下的MYSQL库表升级就会比较麻烦。

        因此本文主要提供了一个脚本，方便比对新旧数据库的sql文件，从而自动生成用户升级的sql语句。

代码

# 两个数据库对比表结构并升级
# 数据库版本更新后，有新增的表，新增的字段
# 通过对比两个库的差异，然后生成语句补充差异

"""
使用说明：使用前请务必阅读理解清楚！！！
1、导出每个待更新新库的结构与数据sql文件，注意如果不需要替换数据的话，可以只导出结构sql文件
2、导出每个待更新旧库的结构与数据sql文件，注意如果不需要替换数据的话，可以只导出结构sql文件
3、给新旧库导出的sql文件内容中的数据库名加上“_new”和“_old”后缀，用于区分，注意是更改文件内容，而不是sql文件名
4、随意找一个能运行python和mysql的电脑，将新旧库的sql文件导入mysql中
5、将python中main函数的sql_info内容修改成上一步使用的mysql的连接信息，并在db_list中填充待更新库的信息，注意table_list是用于数据替换，慎用，会直接清空旧表数据
6、运行python，生成更新用的sql文件
7、检查每个sql文件是否正常
8、在mysql中选择待更新的旧库导入更新用的sql文件测试是否能正常更新
9、将更新用的sql文件拷贝至待更新mysql的服务器上，并依次导入
"""

import pymysql
import datetime


# 获取主键
def getPrimaryKey(cur, database, table):
    sql = f"""
SELECT
    CONSTRAINT_NAME
FROM
    INFORMATION_SCHEMA.TABLE_CONSTRAINTS
WHERE
    CONSTRAINT_SCHEMA = '{database}' AND TABLE_NAME = '{table}';
    """
    cur.execute(sql)
    sql_res = []
    while 1:
        res = cur.fetchone()
        if res is None:
            break  # 表示已经取完结果集s
        sql_res.append(res)
    return sql_res


# 两个数据库相同的表要更新字段的数据类型（db1与db2相同的表和字段，字段的数据类型不同）
def compareColumnTypeInSameDatabaseAndSameTableAndSameColumn(cur, new, old):
    sql = f"""
SELECT DISTINCT
    *
FROM (
    SELECT DISTINCT
        table_name, column_name, column_key, column_comment, column_type, is_nullable, column_default, extra
    FROM
        information_schema.columns
    WHERE
        table_schema = '{new}'
    ) t1
LEFT JOIN (
    SELECT DISTINCT
        table_name, column_name, column_key, column_comment, column_type, is_nullable, column_default, extra
    FROM
        information_schema.columns
    WHERE
        table_schema = '{old}'
    ) t2
ON
    t1.table_name = t2.table_name AND t1.column_name = t2.column_name
WHERE
    t1.column_type != t2.column_type OR t1.is_nullable != t2.is_nullable OR t1.column_default != t2.column_default OR t1.extra != t2.extra
"""
    cur.execute(sql)
    sql_res = []
    while 1:
        res = cur.fetchone()
        if res is None:
            break  # 表示已经取完结果集s
        sql_res.append(res)
    # print('sql_res:', sql_res)
    # 对更新数据类型的sql语句进行拼接
    text_ = ""
    if len(sql_res) == 0:
        return text_
    else:
        sql = f"""
-- 先清除主键约束，避免冲突
SET SQL_REQUIRE_PRIMARY_KEY = OFF;
    """
        text_ += sql + "\n"
        for i in sql_res:
            table_name = i[0]  # 表名TABLE_NAME
            column_name = i[1]  # 字段名COLUMN_NAME
            column_key = i[2]  # 是否为主键COLUMN_KEY
            column_comment = i[3]  # 备注COLUMN_COMMENT
            column_type = i[4]  # 字段类型COLUMN_TYPE
            is_nullable = i[5]  # 字段是否为空IS_NULLABLE
            column_default = i[6]  # 默认值COLUMN_DEFAULT
            extra = i[7]  # 自动递增，更新时间等EXTRA

            # 默认值
            default_ = 'DEFAULT NULL' if column_default is None else f'DEFAULT {column_default}'
            # 字段是否为空
            if is_nullable == 'YES':
                is_nullable_ = 'NULL'
            else:
                is_nullable_ = 'NOT NULL'
                # 当该字段为非空时，判断默认值是否为空，如果是则清除默认值
                default_ = '' if column_default is None else default_
            # 备注
            column_comment_ = f"COMMENT '{column_comment}'" if column_comment else ''
            # 只有主键允许自增
            if extra == 'auto_increment' and column_key == 'PRI':
                column_key_ = 'PRIMARY KEY'
                sql_res = getPrimaryKey(cur, old, table_name)
                if len(sql_res) > 0:
                    sql = f"""
ALTER TABLE
	`{table_name}`
DROP CONSTRAINT
    `{sql_res[0][0]}`;
"""
                    text_ += sql + "\n"
            else:
                column_key_ = ''
            # 额外属性
            if extra == 'auto_increment':
                extra_ = 'AUTO_INCREMENT'
            elif extra == 'DEFAULT_GENERATED':
                extra_ = ''
            elif extra == 'DEFAULT_GENERATED on update CURRENT_TIMESTAMP':
                extra_ = 'ON UPDATE CURRENT_TIMESTAMP'
            else:
                extra_ = extra

            sql = f"""
ALTER TABLE
    `{table_name}`
MODIFY COLUMN
    `{column_name}` {column_type} {column_key_} {is_nullable_} {default_} {extra_} {column_comment_};
"""
            # print(sql)
            text_ = text_ + sql + "\n"
        sql = f"""
-- 恢复主键约束
SET SQL_REQUIRE_PRIMARY_KEY = ON;
"""
        text_ += sql + "\n"
        return text_


# 两个数据库相同表，需要更新的数据库要添加的表字段（db1中的表，db2中存在但是缺少字段）
def compareColumnInSameDatabaseAndSameTable(cur, new, old):
    sql = f"""
SELECT DISTINCT
    *
FROM (
    SELECT DISTINCT
        t1.table_name, t1.column_name, t1.column_type, t1.column_comment, t1.is_nullable, t1.column_key,
        t1.column_default, t1.extra, t1.character_set_name, t1.collation_name
    FROM (
        SELECT DISTINCT
            table_name, column_name, column_comment, column_type, is_nullable, column_key,
            column_default, extra, character_set_name, collation_name
        FROM
            information_schema.columns
        WHERE
            table_schema = '{new}'
    ) t1
LEFT JOIN (
    SELECT DISTINCT
        table_name, column_name, column_comment, column_type
    FROM
        information_schema.columns
    WHERE
        table_schema = '{old}'
    ) t2
ON
    t1.table_name = t2.table_name AND t1.column_name = t2.column_name
WHERE
    t2.table_name IS NULL
) t3
LEFT JOIN (
    SELECT DISTINCT
        t1.table_name
    FROM (
        SELECT DISTINCT
            table_name, column_name, column_comment, column_type
        FROM
            information_schema.columns
        WHERE
            table_schema = '{new}'
    ) t1
LEFT JOIN (
    SELECT DISTINCT
        table_name, column_name, column_comment, column_type
    FROM
        information_schema.columns
    WHERE
        table_schema = '{old}'
    ) t2
ON
    t1.table_name = t2.table_name
WHERE
    t2.table_name IS NULL
) t4
ON
    t3.table_name = t4.table_name
WHERE
    t4.table_name IS NULL
"""
    cur.execute(sql)
    sql_res = []
    while 1:
        res = cur.fetchone()
        if res is None:
            break  # 表示已经取完结果集s
        sql_res.append(res)
    # print('sql_res:', sql_res)
    # 对需要新增字段的sql语句进行拼接.

    text_ = ""
    if len(sql_res) == 0:
        return text_
    else:
        sql = f"""
-- 先清除主键约束，避免冲突
SET SQL_REQUIRE_PRIMARY_KEY = OFF;
    """
        text_ += sql + "\n"
        for i in sql_res:
            table_name = i[0]  # 表名
            column_name = i[1]  # 字段名
            column_type = i[2]  # 字段类型
            column_comment = i[3]  # 备注
            is_nullable = i[4]  # 字段是否为空
            column_key = i[5]  # 字段的主键(PRI为主键)
            column_default = i[6]  # 默认值
            extra = i[7]  # 自动递增，更新时间等
            character_set_name = i[8]  # 字符集
            collation_name = i[9]  # 排序规则
            # 默认值
            default_ = 'DEFAULT NULL' if column_default is None else f'DEFAULT {column_default}'
            # 字段是否为空
            if is_nullable == 'YES':
                is_nullable_ = 'NULL'
            else:
                is_nullable_ = 'NOT NULL'
                # 当该字段为非空时，判断默认值是否为空，如果是则清除默认值
                default_ = '' if column_default is None else default_
            # 只有主键允许自增
            if extra == 'auto_increment' and column_key == 'PRI':
                column_key_ = 'PRIMARY KEY'
                sql_res = getPrimaryKey(cur, old, table_name)
                if len(sql_res) > 0:
                    sql = f"""
ALTER TABLE
	`{table_name}`
DROP CONSTRAINT
    `{sql_res[0][0]}`;
"""
                    text_ += sql + "\n"
            else:
                column_key_ = ''
            # 额外属性
            if extra == 'auto_increment':
                extra_ = 'AUTO_INCREMENT'
            elif extra == 'DEFAULT_GENERATED':
                extra_ = ''
            elif extra == 'DEFAULT_GENERATED on update CURRENT_TIMESTAMP':
                extra_ = 'ON UPDATE CURRENT_TIMESTAMP'
            else:
                extra_ = extra

            # varchar类型的编码
            character_ = f' CHARACTER SET {character_set_name} COLLATE {collation_name}' if character_set_name else ''
            # 备注
            column_comment_ = f" COMMENT '{column_comment}'" if column_comment else ''

            sql = f"""
ALTER TABLE
    `{table_name}`
ADD COLUMN
    `{column_name}` {column_type} {column_key_} {character_} {is_nullable_} {default_} {extra_} {column_comment_};
"""
            # print(sql)
            text_ += sql + "\n"
        sql = f"""
-- 恢复主键约束
SET SQL_REQUIRE_PRIMARY_KEY = ON;
"""
        text_ += sql + "\n"
        return text_


# 需要更新的数据库中需要新建的表（db1中有的表,db2中没有的）
def compareTableInSameDatabase(cur, new, old):
    sql = f"""
SELECT
    table_name
FROM
    information_schema.tables
WHERE
    table_schema = '{new}' AND table_name NOT IN (
        SELECT
            table_name
        FROM
            information_schema.tables
        WHERE
            table_schema = '{old}'
    )
"""
    cur.execute(sql)
    sql_res_1 = []
    while 1:
        res = cur.fetchone()
        if res is None:
            break  # 表示已经取完结果集s
        sql_res_1.append(res)
    # print('sql_res_1:', sql_res_1)
    # 获取新建表的sql建表语句
    text_ = ""
    for i in sql_res_1:
        table_name = i[0]
        cur.execute(f"""SHOW CREATE TABLE {table_name}""")
        sql_res_2 = []
        while 1:
            res = cur.fetchone()
            if res is None:
                break  # 表示已经取完结果集s
            sql_res_2.append(res)
        # print(len(sql_res_2))
        # print(sql_res_2[0][1] + ";")
        text_ += f"""
-- ----------------------------
-- Table structure for {sql_res_2[0][0]}
-- ----------------------------
"""
        text_ += sql_res_2[0][1] + ";\n"
    return text_


# 更新数据
def updateData(cur, new, old, table_list):
    if len(table_list) == 0:
        return ""
    else:
        sql = f"""
SELECT
    *
FROM (
    SELECT DISTINCT
        table_name, column_name, referenced_table_name, referenced_column_name, position_in_unique_constraint
    FROM
        information_schema.key_column_usage
    WHERE
        table_schema = '{new}' AND position_in_unique_constraint = 1
    ) t1
LEFT JOIN (
    SELECT DISTINCT
        table_name, column_name, referenced_table_name, referenced_column_name, position_in_unique_constraint
    FROM
        information_schema.key_column_usage
    WHERE
        table_schema = '{old}' AND position_in_unique_constraint = 1
    ) t2
ON
    t1.table_name = t2.table_name AND t1.column_name = t2.column_name
WHERE
    t2.column_name IS NULL AND t1.table_name NOT IN (
        SELECT
            table_name
        FROM
            information_schema.tables
        WHERE
            table_schema = '{new}' AND table_name NOT IN (
                SELECT
                    table_name
                FROM
                    information_schema.tables
                WHERE
                    table_schema = '{old}'
            )
    )
"""
        cur.execute(sql)
        sql_res = []
        while 1:
            res = cur.fetchone()
            if res is None:
                break  # 表示已经取完结果集
            sql_res.append(res)
        # print('sql_res:', sql_res)
        # 需要更新的数据库中需要补充的外键
        text_= ""
        text_ += """
-- 旧数据库中需要补充的外键
-- 先清除外键约束，等数据更新完成之后再添加外键约束
SET FOREIGN_KEY_CHECKS = 0;\n
"""

        for i in sql_res:
            table_name = i[0]  # 表名
            column_name = i[1]  # 字段名
            referenced_table_name = i[2]  # 关联的表名
            referenced_column_name = i[3]  # 关联的字段名
            position_in_unique_constraint = i[4]  # 关联的字段名
            if position_in_unique_constraint == 1:
                text_ += f"""
ALTER TABLE
    `{table_name}`
ADD FOREIGN KEY (
    `{column_name}`
    )
REFERENCES
    `{referenced_table_name}` (
        `{referenced_column_name}`
    )
ON
    DELETE CASCADE
ON
    UPDATE CASCADE;\n
"""
        # 添加外键约束
        text_ += """
-- 更新完成，添加外键约束
SET FOREIGN_KEY_CHECKS = 1;\n
"""

        # 更新规则，解析器等固定数据表的数据
        text_ += """
-- 更新规则，解析器等固定数据表的数据
-- 覆盖插入数据，先清除外键约束，等数据更新完成之后再添加外键约束
SET FOREIGN_KEY_CHECKS = 0;\n
"""
        # 删除外键约束
        for table_name in table_list:
            text_ += f"""
-- ----------------------------
-- Records of {table_name}
-- ----------------------------
TRUNCATE TABLE {table_name};
"""

            sql = f"""
SELECT
    *
FROM
    {table_name}
"""
            cur.execute(sql)
            field_name_list = '('
            for i, field_name in enumerate(cur.description):
                if i == len(cur.description) - 1:
                    field_name_list += f'{table_name}.{field_name[0]}'
                else:
                    field_name_list += f'{table_name}.{field_name[0]}, '
            field_name_list += ')'
            while 1:
                res = cur.fetchone()
                if res is None:
                    # 表示已经取完结果集
                    break
                new_res = []
                for i in res:
                    if type(i) == datetime.datetime:
                        new_res.append(i.strftime("%Y-%m-%d %H:%M:%S"))
                    else:
                        new_res.append(i)
                text_ += f"""
INSERT INTO
    `{table_name}` {field_name_list}
VALUES
    {tuple(new_res)};\n
""".replace(', None', ', Null')

        # 添加外键约束
        text_ += """
-- 更新完成，添加外键约束
SET FOREIGN_KEY_CHECKS = 1;\n
"""
        return text_


def run(db_list, sql_info):
    pymysql.install_as_MySQLdb()
    for db_info in db_list:
        # 打开数据库连接
        conn = pymysql.connect(
            host=sql_info['sql_host'],
            port=sql_info['sql_port'],
            user=sql_info['sql_user'],
            password=sql_info['sql_pwd'],
            database=db_info['new'],
            charset=sql_info['sql_charset']
        )

        # 获取游标
        cur = conn.cursor()

        # 生成sql语句
        text = ""
        text += compareColumnTypeInSameDatabaseAndSameTableAndSameColumn(cur, db_info['new'], db_info['old'])
        text += compareColumnInSameDatabaseAndSameTable(cur, db_info['new'], db_info['old'])
        text += compareTableInSameDatabase(cur, db_info['new'], db_info['old'])
        text += updateData(cur, db_info['new'], db_info['old'], db_info['table_list']) # 慎用，会清空原来的数据

        # 调用方法写入同目录文件中
        with open("update#" + db_info['name'] + ".sql", "w", encoding="utf-8") as fp:
            fp.write(text)

        cur.close()
        conn.commit()
        conn.close()
        print(db_info['name'] + '的更新sql生成成功')

        # # 执行更新的sql语句
        # conn2 = pymysql.connect(
        #     host=sql_info['sql_host'],
        #     port=sql_info['sql_port'],
        #     user=sql_info['sql_user'],
        #     password=sql_info['sql_pwd'],
        #     database=db_info['old'],
        #     charset=sql_info['sql_charset']
        # )
        # cur2 = conn2.cursor()
        # cur2.execute(text)
        # cur2.close()
        # conn2.commit()
        # conn2.close()
        # print(db_info['name'] + '的更新sql执行成功')


if __name__== "__main__" :
    db_list = [
        {
            'name': "table1",
            # 进行对比的数据库（新库）
            'new': "table1_new",
            # 要更新的数据库（旧库）
            'old': "table1_old",
            # 数据需要覆盖的表，慎用，会清空原来的数据
            'table_list': []
        }
    ]

    sql_info = {
        'sql_host': '127.0.0.1',
        'sql_port': 3306,
        'sql_user': 'root',
        'sql_pwd': 'xxxx', # 根据实际情况修改
        'sql_charset': 'utf8'
    }

    run(db_list, sql_info)

悦读

道可道，非常道；名可名，非常名。 无名，天地之始，有名，万物之母。 故常无欲，以观其妙，常有欲，以观其徼。 此两者，同出而异名，同谓之玄，玄之又玄，众妙之门。

最新收录
QML LinearGradient实现文字渐变闪烁动画
export default function和export function的区别
windows环境下配置libwebsocket
Golang学习日志 ━━ gin-vue-admin换机重新配置的记录，很愚蠢，很傻瓜，很机械...自己使用
学习使用 MATLAB 数学建模一篇文章就够了
用 Python 从单个文本中提取关键字的四种超棒的方法
移动网页唤起手机内 支付宝 微信支付
Linux教程——Linux的主要应用领域有哪些？
离散数学学习笔记-1-
C语言 | Leetcode C语言题解之第45题跳跃游戏II
;