跳转至

Flask-SQLAlchemy 批量导入数据

约 71 个字 59 行代码 预计阅读时间 1 分钟

情境

现在需要导入一系列数据到主表和明细表。实体类定义如下：

实体类
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
class RouteEntity(db.Model):
    __tablename__ = 'routes'

    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
    create_time = db.Column(db.DateTime, default=datetime.now)
    update_time = db.Column(db.DateTime, default=datetime.now)
    create_user = db.Column(db.Integer, db.ForeignKey('users.id'), default=0)
    update_user = db.Column(db.Integer, default=0)
    is_deleted = db.Column(db.Boolean, default=False)
    # ...

    @staticmethod
    def from_dto(route_dto: Route, user_id: int):
        route_points = [RoutePointEntity.from_dto(point, user_id) for point in route_dto.points]
        return RouteEntity(
            points=route_points,
            create_user=user_id,
            update_user=user_id
        )

class RoutePointEntity(db.Model):
    __tablename__ = 'route_points'

    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
    create_time = db.Column(db.DateTime, default=datetime.now)
    update_time = db.Column(db.DateTime, default=datetime.now)
    create_user = db.Column(db.Integer, db.ForeignKey('users.id'), default=0)
    update_user = db.Column(db.Integer, default=0)
    is_deleted = db.Column(db.Boolean, default=False)
    # ...
    route_id = db.Column(db.Integer, db.ForeignKey('routes.id'), nullable=False)
    # ...
    parent_route = db.relationship('RouteEntity', backref=db.backref('points'))

常规做法

数据库操作
1
2
3
route_entity = RouteEntity.from_dto(route, user_id)
db.session.add(route_entity)
db.session.commit()

更快的做法

实体类
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
class RoutePointEntity(db.Model):
    # ...
    def to_bulk_insert_dict(self, route_id: int):
        return {
            'create_time': datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
            'update_time': datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
            'create_user': self.create_user,
            'update_user': self.update_user,
            'route_id': route_id,
            # ...
        }
数据库操作
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
route_entity = RouteEntity.from_dto(route)
route_entity_bare = RouteEntity(
    create_user=user_id,
    update_user=user_id
    # 相比于 route_entity，这里面唯独不传 points
)
db.session.add(route_entity_bare)
db.session.flush()  # 获取route_entity的ID，但不提交事务
for i in route_entity.points:
    i.route_id = route_entity.id
db.session.execute(insert(RoutePointEntity), [i.to_bulk_insert_dict(route_entity_bare.id) for i in route_entity.points])
db.session.commit()

实测效果

使用 MySQL，当子表要插入 17849 条数据时，时间从 14 s 缩减到 5 s。