mirror of
https://github.com/GNS3/gns3-server
synced 2025-01-13 17:40:54 +00:00
Merge pull request #2160 from GNS3/db-migrations
Support for database schema migrations
This commit is contained in:
commit
ce3bf9930c
103
gns3server/alembic.ini
Normal file
103
gns3server/alembic.ini
Normal file
@ -0,0 +1,103 @@
|
|||||||
|
# A generic, single database configuration.
|
||||||
|
|
||||||
|
[alembic]
|
||||||
|
# path to migration scripts
|
||||||
|
script_location = db_migrations
|
||||||
|
|
||||||
|
# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
|
||||||
|
# Uncomment the line below if you want the files to be prepended with date and time
|
||||||
|
# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
|
||||||
|
|
||||||
|
# sys.path path, will be prepended to sys.path if present.
|
||||||
|
# defaults to the current working directory.
|
||||||
|
prepend_sys_path = .
|
||||||
|
|
||||||
|
# timezone to use when rendering the date within the migration file
|
||||||
|
# as well as the filename.
|
||||||
|
# If specified, requires the python-dateutil library that can be
|
||||||
|
# installed by adding `alembic[tz]` to the pip requirements
|
||||||
|
# string value is passed to dateutil.tz.gettz()
|
||||||
|
# leave blank for localtime
|
||||||
|
# timezone =
|
||||||
|
|
||||||
|
# max length of characters to apply to the
|
||||||
|
# "slug" field
|
||||||
|
# truncate_slug_length = 40
|
||||||
|
|
||||||
|
# set to 'true' to run the environment during
|
||||||
|
# the 'revision' command, regardless of autogenerate
|
||||||
|
# revision_environment = false
|
||||||
|
|
||||||
|
# set to 'true' to allow .pyc and .pyo files without
|
||||||
|
# a source .py file to be detected as revisions in the
|
||||||
|
# versions/ directory
|
||||||
|
# sourceless = false
|
||||||
|
|
||||||
|
# version location specification; This defaults
|
||||||
|
# to db_migrations/versions. When using multiple version
|
||||||
|
# directories, initial revisions must be specified with --version-path.
|
||||||
|
# The path separator used here should be the separator specified by "version_path_separator" below.
|
||||||
|
# version_locations = %(here)s/bar:%(here)s/bat:db_migrations/versions
|
||||||
|
|
||||||
|
# version path separator; As mentioned above, this is the character used to split
|
||||||
|
# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
|
||||||
|
# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
|
||||||
|
# Valid values for version_path_separator are:
|
||||||
|
#
|
||||||
|
# version_path_separator = :
|
||||||
|
# version_path_separator = ;
|
||||||
|
# version_path_separator = space
|
||||||
|
version_path_separator = os # Use os.pathsep. Default configuration used for new projects.
|
||||||
|
|
||||||
|
# the output encoding used when revision files
|
||||||
|
# are written from script.py.mako
|
||||||
|
# output_encoding = utf-8
|
||||||
|
|
||||||
|
sqlalchemy.url =
|
||||||
|
|
||||||
|
|
||||||
|
[post_write_hooks]
|
||||||
|
# post_write_hooks defines scripts or Python functions that are run
|
||||||
|
# on newly generated revision scripts. See the documentation for further
|
||||||
|
# detail and examples
|
||||||
|
|
||||||
|
# format using "black" - use the console_scripts runner, against the "black" entrypoint
|
||||||
|
# hooks = black
|
||||||
|
# black.type = console_scripts
|
||||||
|
# black.entrypoint = black
|
||||||
|
# black.options = -l 79 REVISION_SCRIPT_FILENAME
|
||||||
|
|
||||||
|
# Logging configuration
|
||||||
|
[loggers]
|
||||||
|
keys = root,sqlalchemy,alembic
|
||||||
|
|
||||||
|
[handlers]
|
||||||
|
keys = console
|
||||||
|
|
||||||
|
[formatters]
|
||||||
|
keys = generic
|
||||||
|
|
||||||
|
[logger_root]
|
||||||
|
level = WARN
|
||||||
|
handlers = console
|
||||||
|
qualname =
|
||||||
|
|
||||||
|
[logger_sqlalchemy]
|
||||||
|
level = WARN
|
||||||
|
handlers =
|
||||||
|
qualname = sqlalchemy.engine
|
||||||
|
|
||||||
|
[logger_alembic]
|
||||||
|
level = INFO
|
||||||
|
handlers =
|
||||||
|
qualname = alembic
|
||||||
|
|
||||||
|
[handler_console]
|
||||||
|
class = StreamHandler
|
||||||
|
args = (sys.stderr,)
|
||||||
|
level = NOTSET
|
||||||
|
formatter = generic
|
||||||
|
|
||||||
|
[formatter_generic]
|
||||||
|
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||||
|
datefmt = %H:%M:%S
|
@ -28,6 +28,10 @@ from sqlalchemy import event
|
|||||||
from sqlalchemy.engine import Engine
|
from sqlalchemy.engine import Engine
|
||||||
from sqlalchemy.exc import SQLAlchemyError
|
from sqlalchemy.exc import SQLAlchemyError
|
||||||
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
|
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
|
||||||
|
from alembic import command, config
|
||||||
|
from alembic.script import ScriptDirectory
|
||||||
|
from alembic.runtime.migration import MigrationContext
|
||||||
|
|
||||||
from gns3server.db.repositories.computes import ComputesRepository
|
from gns3server.db.repositories.computes import ComputesRepository
|
||||||
from gns3server.db.repositories.images import ImagesRepository
|
from gns3server.db.repositories.images import ImagesRepository
|
||||||
from gns3server.utils.images import discover_images, check_valid_image_header, read_image_info, default_images_directory, InvalidImageError
|
from gns3server.utils.images import discover_images, check_valid_image_header, read_image_info, default_images_directory, InvalidImageError
|
||||||
@ -41,15 +45,46 @@ import logging
|
|||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def run_upgrade(connection, cfg):
|
||||||
|
|
||||||
|
cfg.attributes["connection"] = connection
|
||||||
|
command.upgrade(cfg, "head")
|
||||||
|
|
||||||
|
|
||||||
|
def run_stamp(connection, cfg):
|
||||||
|
|
||||||
|
cfg.attributes["connection"] = connection
|
||||||
|
command.stamp(cfg, 'head')
|
||||||
|
|
||||||
|
|
||||||
|
def check_revision(connection, cfg):
|
||||||
|
|
||||||
|
script = ScriptDirectory.from_config(cfg)
|
||||||
|
head_rev = script.get_revision("head").revision
|
||||||
|
context = MigrationContext.configure(connection)
|
||||||
|
current_rev = context.get_current_revision()
|
||||||
|
return current_rev, head_rev
|
||||||
|
|
||||||
|
|
||||||
async def connect_to_db(app: FastAPI) -> None:
|
async def connect_to_db(app: FastAPI) -> None:
|
||||||
|
|
||||||
db_path = os.path.join(Config.instance().config_dir, "gns3_controller.db")
|
db_path = os.path.join(Config.instance().config_dir, "gns3_controller.db")
|
||||||
db_url = os.environ.get("GNS3_DATABASE_URI", f"sqlite+aiosqlite:///{db_path}")
|
db_url = os.environ.get("GNS3_DATABASE_URI", f"sqlite+aiosqlite:///{db_path}")
|
||||||
engine = create_async_engine(db_url, connect_args={"check_same_thread": False}, future=True)
|
engine = create_async_engine(db_url, connect_args={"check_same_thread": False}, future=True)
|
||||||
|
alembic_cfg = config.Config()
|
||||||
|
alembic_cfg.set_main_option("script_location", "gns3server:db_migrations")
|
||||||
|
#alembic_cfg.set_main_option('sqlalchemy.url', db_url)
|
||||||
try:
|
try:
|
||||||
async with engine.connect() as conn:
|
async with engine.connect() as conn:
|
||||||
await conn.run_sync(Base.metadata.create_all)
|
current_rev, head_rev = await conn.run_sync(check_revision, alembic_cfg)
|
||||||
log.info(f"Successfully connected to database '{db_url}'")
|
log.info(f"Current database revision is {current_rev}")
|
||||||
|
if current_rev is None:
|
||||||
|
await conn.run_sync(Base.metadata.create_all)
|
||||||
|
await conn.run_sync(run_stamp, alembic_cfg)
|
||||||
|
elif current_rev != head_rev:
|
||||||
|
# upgrade the database if needed
|
||||||
|
await conn.run_sync(run_upgrade, alembic_cfg)
|
||||||
|
await conn.commit()
|
||||||
app.state._db_engine = engine
|
app.state._db_engine = engine
|
||||||
except SQLAlchemyError as e:
|
except SQLAlchemyError as e:
|
||||||
log.fatal(f"Error while connecting to database '{db_url}: {e}")
|
log.fatal(f"Error while connecting to database '{db_url}: {e}")
|
||||||
|
4
gns3server/db_migrations/README
Normal file
4
gns3server/db_migrations/README
Normal file
@ -0,0 +1,4 @@
|
|||||||
|
Generic single-database configuration with an async dbapi.
|
||||||
|
|
||||||
|
# Command to generate a revision
|
||||||
|
alembic revision --autogenerate -m "add fields in table"
|
101
gns3server/db_migrations/env.py
Normal file
101
gns3server/db_migrations/env.py
Normal file
@ -0,0 +1,101 @@
|
|||||||
|
import asyncio
|
||||||
|
import os
|
||||||
|
from logging.config import fileConfig
|
||||||
|
|
||||||
|
from sqlalchemy import pool
|
||||||
|
from sqlalchemy.ext.asyncio import async_engine_from_config
|
||||||
|
|
||||||
|
from alembic import context
|
||||||
|
|
||||||
|
# this is the Alembic Config object, which provides
|
||||||
|
# access to the values within the .ini file in use.
|
||||||
|
config = context.config
|
||||||
|
|
||||||
|
# Interpret the config file for Python logging.
|
||||||
|
# This line sets up loggers basically.
|
||||||
|
if config.config_file_name is not None:
|
||||||
|
fileConfig(config.config_file_name)
|
||||||
|
|
||||||
|
# add your model's MetaData object here
|
||||||
|
# for 'autogenerate' support
|
||||||
|
# from myapp import mymodel
|
||||||
|
# target_metadata = mymodel.Base.metadata
|
||||||
|
from gns3server.db.models.base import Base
|
||||||
|
target_metadata = Base.metadata
|
||||||
|
|
||||||
|
# other values from the config, defined by the needs of env.py,
|
||||||
|
# can be acquired:
|
||||||
|
# my_important_option = config.get_main_option("my_important_option")
|
||||||
|
# ... etc.
|
||||||
|
|
||||||
|
|
||||||
|
def run_migrations_offline() -> None:
|
||||||
|
"""Run migrations in 'offline' mode.
|
||||||
|
|
||||||
|
This configures the context with just a URL
|
||||||
|
and not an Engine, though an Engine is acceptable
|
||||||
|
here as well. By skipping the Engine creation
|
||||||
|
we don't even need a DBAPI to be available.
|
||||||
|
|
||||||
|
Calls to context.execute() here emit the given string to the
|
||||||
|
script output.
|
||||||
|
|
||||||
|
"""
|
||||||
|
url = config.get_main_option("sqlalchemy.url")
|
||||||
|
context.configure(
|
||||||
|
url=url,
|
||||||
|
target_metadata=target_metadata,
|
||||||
|
literal_binds=True,
|
||||||
|
dialect_opts={"paramstyle": "named"},
|
||||||
|
)
|
||||||
|
|
||||||
|
with context.begin_transaction():
|
||||||
|
context.run_migrations()
|
||||||
|
|
||||||
|
|
||||||
|
def do_run_migrations(connection):
|
||||||
|
context.configure(connection=connection, target_metadata=target_metadata)
|
||||||
|
|
||||||
|
with context.begin_transaction():
|
||||||
|
context.run_migrations()
|
||||||
|
|
||||||
|
|
||||||
|
async def run_async_migrations():
|
||||||
|
|
||||||
|
from gns3server.config import Config as GNS3Config
|
||||||
|
db_path = os.path.join(GNS3Config.instance().config_dir, "gns3_controller.db")
|
||||||
|
db_url = os.environ.get("GNS3_DATABASE_URI", f"sqlite+aiosqlite:///{db_path}")
|
||||||
|
config.set_main_option('sqlalchemy.url', db_url)
|
||||||
|
|
||||||
|
connectable = async_engine_from_config(
|
||||||
|
config.get_section(config.config_ini_section),
|
||||||
|
prefix="sqlalchemy.",
|
||||||
|
poolclass=pool.NullPool,
|
||||||
|
)
|
||||||
|
|
||||||
|
async with connectable.connect() as connection:
|
||||||
|
await connection.run_sync(do_run_migrations)
|
||||||
|
|
||||||
|
await connectable.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
def run_migrations_online():
|
||||||
|
"""Run migrations in 'online' mode.
|
||||||
|
|
||||||
|
In this scenario we need to create an Engine
|
||||||
|
and associate a connection with the context.
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
connectable = config.attributes.get("connection", None)
|
||||||
|
|
||||||
|
if connectable is None:
|
||||||
|
asyncio.run(run_async_migrations())
|
||||||
|
else:
|
||||||
|
do_run_migrations(connectable)
|
||||||
|
|
||||||
|
|
||||||
|
if context.is_offline_mode():
|
||||||
|
run_migrations_offline()
|
||||||
|
else:
|
||||||
|
run_migrations_online()
|
24
gns3server/db_migrations/script.py.mako
Normal file
24
gns3server/db_migrations/script.py.mako
Normal file
@ -0,0 +1,24 @@
|
|||||||
|
"""${message}
|
||||||
|
|
||||||
|
Revision ID: ${up_revision}
|
||||||
|
Revises: ${down_revision | comma,n}
|
||||||
|
Create Date: ${create_date}
|
||||||
|
|
||||||
|
"""
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
${imports if imports else ""}
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision = ${repr(up_revision)}
|
||||||
|
down_revision = ${repr(down_revision)}
|
||||||
|
branch_labels = ${repr(branch_labels)}
|
||||||
|
depends_on = ${repr(depends_on)}
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
${upgrades if upgrades else "pass"}
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
${downgrades if downgrades else "pass"}
|
28
gns3server/db_migrations/versions/7ceeddd9c9a8_init.py
Normal file
28
gns3server/db_migrations/versions/7ceeddd9c9a8_init.py
Normal file
@ -0,0 +1,28 @@
|
|||||||
|
"""init
|
||||||
|
|
||||||
|
Revision ID: 7ceeddd9c9a8
|
||||||
|
Revises:
|
||||||
|
Create Date: 2023-01-11 08:55:25.116126
|
||||||
|
|
||||||
|
"""
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision = '7ceeddd9c9a8'
|
||||||
|
down_revision = None
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
# ### commands auto generated by Alembic - please adjust! ###
|
||||||
|
pass
|
||||||
|
# ### end Alembic commands ###
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
# ### commands auto generated by Alembic - please adjust! ###
|
||||||
|
pass
|
||||||
|
# ### end Alembic commands ###
|
@ -12,6 +12,7 @@ distro>=1.7.0
|
|||||||
py-cpuinfo==9.0.0
|
py-cpuinfo==9.0.0
|
||||||
sqlalchemy==1.4.45
|
sqlalchemy==1.4.45
|
||||||
aiosqlite==0.18.0
|
aiosqlite==0.18.0
|
||||||
|
alembic==1.9.1
|
||||||
passlib[bcrypt]==1.7.4
|
passlib[bcrypt]==1.7.4
|
||||||
python-jose==3.3.0
|
python-jose==3.3.0
|
||||||
email-validator==1.3.0
|
email-validator==1.3.0
|
||||||
|
Loading…
Reference in New Issue
Block a user